From 6dc2c16f95c7b9a7f33964d2946ba2f8a6e7de9b Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 22 Dec 2020 16:31:56 -0500 Subject: [PATCH 01/25] [dev.cmdgo] codereview.cfg: add config for dev.cmdgo Change-Id: I6a711402b06a75c5cba43a72950617fea27bd50b Reviewed-on: https://go-review.googlesource.com/c/go/+/279526 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Russ Cox --- codereview.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 codereview.cfg diff --git a/codereview.cfg b/codereview.cfg new file mode 100644 index 00000000000..c5bef5e62d2 --- /dev/null +++ b/codereview.cfg @@ -0,0 +1,2 @@ +branch: dev.cmdgo +parent-branch: master From ab361499ef7fc7079c78b566f9ff7d68c267b430 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 7 Jul 2021 19:41:02 -0400 Subject: [PATCH 02/25] [dev.cmdgo] cmd/go/testdata/script: fix a small typo in modfile_flag Change-Id: Id854869e581645dad7a250d40b150ebaf541c043 Reviewed-on: https://go-review.googlesource.com/c/go/+/334931 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/testdata/script/modfile_flag.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/go/testdata/script/modfile_flag.txt b/src/cmd/go/testdata/script/modfile_flag.txt index 0ad08808178..5852c311891 100644 --- a/src/cmd/go/testdata/script/modfile_flag.txt +++ b/src/cmd/go/testdata/script/modfile_flag.txt @@ -73,7 +73,7 @@ cmp go.mod go.mod.orig cmp go.sum go.sum.orig -# If the altnernate mod file does not have a ".mod" suffix, an error +# If the alternate mod file does not have a ".mod" suffix, an error # should be reported. cp go.alt.mod goaltmod ! go mod tidy -modfile=goaltmod From a627fcd3c4fdacdc9bbcccdb926e4804ca6d6815 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Fri, 14 May 2021 11:46:26 -0400 Subject: [PATCH 03/25] [dev.cmdgo] cmd/go: replace Target with MainModules, allowing for multiple targets This change replaces the Target variable that represents the main module and the pathPrefix and inGorootSrc which provide other information about the main module with a single MainModules value that represents multiple main modules and holds their path prefixes, module roots, and whether they are in GOROOT/src. In cases where the code checks Target or its previously associated variables, the code now checks or iterates over MainModules. In some cases, the code still assumes a single main module by calling MainModules.MustGetSingleMainModule. Some of those cases are correct: for instance, there is always only one main module for mod=vendor. Other cases are accompanied with TODOs and will have to be fixed in future CLs to properly support multiple main modules. This CL (and other cls on top of it) are planned to be checked into a branch to allow for those evaluating the workspaces proposal to try it hands on. For #45713 Change-Id: I3b699e1d5cad8c76d62dc567b8460de8c73a87ea Reviewed-on: https://go-review.googlesource.com/c/go/+/334932 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/get/get.go | 6 +- src/cmd/go/internal/load/pkg.go | 7 +- src/cmd/go/internal/modcmd/download.go | 14 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modget/get.go | 50 ++-- src/cmd/go/internal/modget/query.go | 6 +- src/cmd/go/internal/modload/build.go | 19 +- src/cmd/go/internal/modload/buildlist.go | 77 +++--- src/cmd/go/internal/modload/edit.go | 20 +- src/cmd/go/internal/modload/import.go | 24 +- src/cmd/go/internal/modload/init.go | 286 ++++++++++++++++------- src/cmd/go/internal/modload/list.go | 6 +- src/cmd/go/internal/modload/load.go | 152 +++++++----- src/cmd/go/internal/modload/modfile.go | 12 +- src/cmd/go/internal/modload/mvs.go | 4 +- src/cmd/go/internal/modload/query.go | 111 +++++---- src/cmd/go/internal/modload/search.go | 15 +- src/cmd/go/internal/modload/vendor.go | 1 + src/cmd/go/internal/mvs/mvs.go | 34 +-- src/cmd/go/internal/mvs/mvs_test.go | 2 +- src/cmd/go/internal/search/search.go | 33 +-- 21 files changed, 565 insertions(+), 316 deletions(-) diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index 3c16dc3040f..cc676428e24 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -225,7 +225,8 @@ func downloadPaths(patterns []string) []string { base.ExitIfErrors() var pkgs []string - for _, m := range search.ImportPathsQuiet(patterns) { + noModRoots := []string{} + for _, m := range search.ImportPathsQuiet(patterns, noModRoots) { if len(m.Pkgs) == 0 && strings.Contains(m.Pattern(), "...") { pkgs = append(pkgs, m.Pattern()) } else { @@ -315,7 +316,8 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) if wildcardOkay && strings.Contains(arg, "...") { match := search.NewMatch(arg) if match.IsLocal() { - match.MatchDirs() + noModRoots := []string{} // We're in gopath mode, so there are no modroots. + match.MatchDirs(noModRoots) args = match.Dirs } else { match.MatchPackages() diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index a83cc9a812b..ac89127a4b1 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1450,9 +1450,9 @@ func disallowInternal(ctx context.Context, srcDir string, importer *Package, imp // The importer is a list of command-line files. // Pretend that the import path is the import path of the // directory containing them. - // If the directory is outside the main module, this will resolve to ".", + // If the directory is outside the main modules, this will resolve to ".", // which is not a prefix of any valid module. - importerPath = modload.DirImportPath(ctx, importer.Dir) + importerPath, _ = modload.MainModules.DirImportPath(ctx, importer.Dir) } parentOfInternal := p.ImportPath[:i] if str.HasPathPrefix(importerPath, parentOfInternal) { @@ -2447,7 +2447,8 @@ func PackagesAndErrors(ctx context.Context, opts PackageOpts, patterns []string) } matches, _ = modload.LoadPackages(ctx, modOpts, patterns...) } else { - matches = search.ImportPaths(patterns) + noModRoots := []string{} + matches = search.ImportPaths(patterns, noModRoots) } var ( diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 0e5af852376..3c88a4b900e 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -91,12 +91,18 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { args = []string{"all"} } if modload.HasModRoot() { - modload.LoadModFile(ctx) // to fill Target - targetAtUpgrade := modload.Target.Path + "@upgrade" - targetAtPatch := modload.Target.Path + "@patch" + modload.LoadModFile(ctx) // to fill MainModules + + if len(modload.MainModules.Versions()) != 1 { + panic(modload.TODOWorkspaces("TODO: multiple main modules not supported in Download")) + } + mainModule := modload.MainModules.Versions()[0] + + targetAtUpgrade := mainModule.Path + "@upgrade" + targetAtPatch := mainModule.Path + "@patch" for _, arg := range args { switch arg { - case modload.Target.Path, targetAtUpgrade, targetAtPatch: + case mainModule.Path, targetAtUpgrade, targetAtPatch: os.Stderr.WriteString("go mod download: skipping argument " + arg + " that resolves to the main module\n") } } diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 713d5f9f3fa..3506655a4a5 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -82,7 +82,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { modpkgs := make(map[module.Version][]string) for _, pkg := range pkgs { m := modload.PackageModule(pkg) - if m.Path == "" || m == modload.Target { + if m.Path == "" || m.Version == "" && modload.MainModules.Contains(m.Path) { continue } modpkgs[m] = append(modpkgs[m], pkg) diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 9672e5598e0..3c8b5a70900 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -389,9 +389,11 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { haveExternalExe := false for _, pkg := range pkgs { - if pkg.Name == "main" && pkg.Module != nil && pkg.Module.Path != modload.Target.Path { - haveExternalExe = true - break + if pkg.Name == "main" && pkg.Module != nil { + if !modload.MainModules.Contains(pkg.Module.Path) { + haveExternalExe = true + break + } } } if haveExternalExe { @@ -675,7 +677,9 @@ func (r *resolver) queryNone(ctx context.Context, q *query) { if !q.isWildcard() { q.pathOnce(q.pattern, func() pathSet { - if modload.HasModRoot() && q.pattern == modload.Target.Path { + hasModRoot := modload.HasModRoot() + if hasModRoot && modload.MainModules.Contains(q.pattern) { + v := module.Version{Path: q.pattern} // The user has explicitly requested to downgrade their own module to // version "none". This is not an entirely unreasonable request: it // could plausibly mean “downgrade away everything that depends on any @@ -686,7 +690,7 @@ func (r *resolver) queryNone(ctx context.Context, q *query) { // However, neither of those behaviors would be consistent with the // plain meaning of the query. To try to reduce confusion, reject the // query explicitly. - return errSet(&modload.QueryMatchesMainModuleError{Pattern: q.pattern, Query: q.version}) + return errSet(&modload.QueryMatchesMainModuleError{MainModule: v, Pattern: q.pattern, Query: q.version}) } return pathSet{mod: module.Version{Path: q.pattern, Version: "none"}} @@ -698,8 +702,8 @@ func (r *resolver) queryNone(ctx context.Context, q *query) { continue } q.pathOnce(curM.Path, func() pathSet { - if modload.HasModRoot() && curM == modload.Target { - return errSet(&modload.QueryMatchesMainModuleError{Pattern: q.pattern, Query: q.version}) + if modload.HasModRoot() && curM.Version == "" && modload.MainModules.Contains(curM.Path) { + return errSet(&modload.QueryMatchesMainModuleError{MainModule: curM, Pattern: q.pattern, Query: q.version}) } return pathSet{mod: module.Version{Path: curM.Path, Version: "none"}} }) @@ -718,12 +722,12 @@ func (r *resolver) performLocalQueries(ctx context.Context) { // Absolute paths like C:\foo and relative paths like ../foo... are // restricted to matching packages in the main module. - pkgPattern := modload.DirImportPath(ctx, q.pattern) + pkgPattern, mainModule := modload.MainModules.DirImportPath(ctx, q.pattern) if pkgPattern == "." { return errSet(fmt.Errorf("%s%s is not within module rooted at %s", q.pattern, absDetail, modload.ModRoot())) } - match := modload.MatchInModule(ctx, pkgPattern, modload.Target, imports.AnyTags()) + match := modload.MatchInModule(ctx, pkgPattern, mainModule, imports.AnyTags()) if len(match.Errs) > 0 { return pathSet{err: match.Errs[0]} } @@ -739,7 +743,7 @@ func (r *resolver) performLocalQueries(ctx context.Context) { return pathSet{} } - return pathSet{pkgMods: []module.Version{modload.Target}} + return pathSet{pkgMods: []module.Version{mainModule}} }) } } @@ -789,11 +793,12 @@ func (r *resolver) queryWildcard(ctx context.Context, q *query) { return pathSet{} } - if curM.Path == modload.Target.Path && !versionOkForMainModule(q.version) { + if modload.MainModules.Contains(curM.Path) && !versionOkForMainModule(q.version) { if q.matchesPath(curM.Path) { return errSet(&modload.QueryMatchesMainModuleError{ - Pattern: q.pattern, - Query: q.version, + MainModule: curM, + Pattern: q.pattern, + Query: q.version, }) } @@ -1159,8 +1164,8 @@ func (r *resolver) loadPackages(ctx context.Context, patterns []string, findPack } opts.AllowPackage = func(ctx context.Context, path string, m module.Version) error { - if m.Path == "" || m == modload.Target { - // Packages in the standard library and main module are already at their + if m.Path == "" || m.Version == "" && modload.MainModules.Contains(m.Path) { + // Packages in the standard library and main modules are already at their // latest (and only) available versions. return nil } @@ -1370,11 +1375,11 @@ func (r *resolver) disambiguate(cs pathSet) (filtered pathSet, isPackage bool, m continue } - if m.Path == modload.Target.Path { - if m.Version == modload.Target.Version { + if modload.MainModules.Contains(m.Path) { + if m.Version == "" { return pathSet{}, true, m, true } - // The main module can only be set to its own version. + // A main module can only be set to its own version. continue } @@ -1744,10 +1749,11 @@ func (r *resolver) resolve(q *query, m module.Version) { panic("internal error: resolving a module.Version with an empty path") } - if m.Path == modload.Target.Path && m.Version != modload.Target.Version { + if modload.MainModules.Contains(m.Path) && m.Version != "" { reportError(q, &modload.QueryMatchesMainModuleError{ - Pattern: q.pattern, - Query: q.version, + MainModule: module.Version{Path: m.Path}, + Pattern: q.pattern, + Query: q.version, }) return } @@ -1775,7 +1781,7 @@ func (r *resolver) updateBuildList(ctx context.Context, additions []module.Versi resolved := make([]module.Version, 0, len(r.resolvedVersion)) for mPath, rv := range r.resolvedVersion { - if mPath != modload.Target.Path { + if !modload.MainModules.Contains(mPath) { resolved = append(resolved, module.Version{Path: mPath, Version: rv.version}) } } diff --git a/src/cmd/go/internal/modget/query.go b/src/cmd/go/internal/modget/query.go index 1a5a60f7eb9..bbb364fa5ee 100644 --- a/src/cmd/go/internal/modget/query.go +++ b/src/cmd/go/internal/modget/query.go @@ -192,9 +192,9 @@ func (q *query) validate() error { // TODO(bcmills): "all@none" seems like a totally reasonable way to // request that we remove all module requirements, leaving only the main // module and standard library. Perhaps we should implement that someday. - return &modload.QueryMatchesMainModuleError{ - Pattern: q.pattern, - Query: q.version, + return &modload.QueryUpgradesAllError{ + MainModules: modload.MainModules.Versions(), + Query: q.version, } } } diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 76e1ad589f4..a5c5ad9b442 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -212,20 +212,21 @@ func addDeprecation(ctx context.Context, m *modinfo.ModulePublic) { // in rs (which may be nil to indicate that m was not loaded from a requirement // graph). func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode ListMode) *modinfo.ModulePublic { - if m == Target { + if m.Version == "" && MainModules.Contains(m.Path) { info := &modinfo.ModulePublic{ Path: m.Path, Version: m.Version, Main: true, } - if v, ok := rawGoVersion.Load(Target); ok { + _ = TODOWorkspaces("handle rawGoVersion here") + if v, ok := rawGoVersion.Load(m); ok { info.GoVersion = v.(string) } else { panic("internal error: GoVersion not set for main module") } - if HasModRoot() { - info.Dir = ModRoot() - info.GoMod = ModFilePath() + if modRoot := MainModules.ModRoot(m); modRoot != "" { + info.Dir = modRoot + info.GoMod = modFilePath(modRoot) } return info } @@ -397,7 +398,8 @@ func mustFindModule(ld *loader, target, path string) module.Version { } if path == "command-line-arguments" { - return Target + _ = TODOWorkspaces("support multiple main modules; search by modroot") + return MainModules.mustGetSingleMainModule() } base.Fatalf("build %v: cannot find module for path %v", target, path) @@ -406,13 +408,14 @@ func mustFindModule(ld *loader, target, path string) module.Version { // findModule searches for the module that contains the package at path. // If the package was loaded, its containing module and true are returned. -// Otherwise, module.Version{} and false are returend. +// Otherwise, module.Version{} and false are returned. func findModule(ld *loader, path string) (module.Version, bool) { if pkg, ok := ld.pkgCache.Get(path).(*loadPkg); ok { return pkg.mod, pkg.mod != module.Version{} } if path == "command-line-arguments" { - return Target, true + _ = TODOWorkspaces("support multiple main modules; search by modroot") + return MainModules.mustGetSingleMainModule(), true } return module.Version{}, false } diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 604a57b4373..959ee25df4e 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -40,7 +40,7 @@ type Requirements struct { depth modDepth // rootModules is the set of module versions explicitly required by the main - // module, sorted and capped to length. It may contain duplicates, and may + // modules, sorted and capped to length. It may contain duplicates, and may // contain multiple versions for a given module path. rootModules []module.Version maxRootVersion map[string]string @@ -99,8 +99,8 @@ var requirements *Requirements // *Requirements before any other method. func newRequirements(depth modDepth, rootModules []module.Version, direct map[string]bool) *Requirements { for i, m := range rootModules { - if m == Target { - panic(fmt.Sprintf("newRequirements called with untrimmed build list: rootModules[%v] is Target", i)) + if m.Version == "" && MainModules.Contains(m.Path) { + panic(fmt.Sprintf("newRequirements called with untrimmed build list: rootModules[%v] is a main module", i)) } if m.Path == "" || m.Version == "" { panic(fmt.Sprintf("bad requirement: rootModules[%v] = %v", i, m)) @@ -135,9 +135,14 @@ func newRequirements(depth modDepth, rootModules []module.Version, direct map[st func (rs *Requirements) initVendor(vendorList []module.Version) { rs.graphOnce.Do(func() { mg := &ModuleGraph{ - g: mvs.NewGraph(cmpVersion, []module.Version{Target}), + g: mvs.NewGraph(cmpVersion, MainModules.Versions()), } + if MainModules.Len() != 1 { + panic("There should be exactly one main moudle in Vendor mode.") + } + mainModule := MainModules.Versions()[0] + if rs.depth == lazy { // The roots of a lazy module should already include every module in the // vendor list, because the vendored modules are the same as those @@ -158,7 +163,7 @@ func (rs *Requirements) initVendor(vendorList []module.Version) { // Now we can treat the rest of the module graph as effectively “pruned // out”, like a more aggressive version of lazy loading: in vendor mode, // the root requirements *are* the complete module graph. - mg.g.Require(Target, rs.rootModules) + mg.g.Require(mainModule, rs.rootModules) } else { // The transitive requirements of the main module are not in general available // from the vendor directory, and we don't actually know how we got from @@ -170,7 +175,7 @@ func (rs *Requirements) initVendor(vendorList []module.Version) { // graph, but still distinguishes between direct and indirect // dependencies. vendorMod := module.Version{Path: "vendor/modules.txt", Version: ""} - mg.g.Require(Target, append(rs.rootModules, vendorMod)) + mg.g.Require(mainModule, append(rs.rootModules, vendorMod)) mg.g.Require(vendorMod, vendorList) } @@ -182,8 +187,8 @@ func (rs *Requirements) initVendor(vendorList []module.Version) { // path, or the zero module.Version and ok=false if the module is not a root // dependency. func (rs *Requirements) rootSelected(path string) (version string, ok bool) { - if path == Target.Path { - return Target.Version, true + if MainModules.Contains(path) { + return "", true } if v, ok := rs.maxRootVersion[path]; ok { return v, true @@ -261,10 +266,15 @@ func readModGraph(ctx context.Context, depth modDepth, roots []module.Version) ( mu sync.Mutex // guards mg.g and hasError during loading hasError bool mg = &ModuleGraph{ - g: mvs.NewGraph(cmpVersion, []module.Version{Target}), + g: mvs.NewGraph(cmpVersion, MainModules.Versions()), } ) - mg.g.Require(Target, roots) + for _, m := range MainModules.Versions() { + // Require all roots from all main modules. + _ = TODOWorkspaces("This isn't the correct behavior. " + + "Fix this when the requirements struct is updated to reflect the struct of the module graph.") + mg.g.Require(m, roots) + } var ( loadQueue = par.NewQueue(runtime.GOMAXPROCS(0)) @@ -391,10 +401,12 @@ func (mg *ModuleGraph) findError() error { } func (mg *ModuleGraph) allRootsSelected() bool { - roots, _ := mg.g.RequiredBy(Target) - for _, m := range roots { - if mg.Selected(m.Path) != m.Version { - return false + for _, mm := range MainModules.Versions() { + roots, _ := mg.g.RequiredBy(mm) + for _, m := range roots { + if mg.Selected(m.Path) != m.Version { + return false + } } } return true @@ -533,10 +545,11 @@ type Conflict struct { // both retain the same versions of all packages in pkgs and satisfy the // lazy loading invariants (if applicable). func tidyRoots(ctx context.Context, rs *Requirements, pkgs []*loadPkg) (*Requirements, error) { + mainModule := MainModules.mustGetSingleMainModule() if rs.depth == eager { - return tidyEagerRoots(ctx, rs.direct, pkgs) + return tidyEagerRoots(ctx, mainModule, rs.direct, pkgs) } - return tidyLazyRoots(ctx, rs.direct, pkgs) + return tidyLazyRoots(ctx, mainModule, rs.direct, pkgs) } func updateRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version, rootsImported bool) (*Requirements, error) { @@ -561,10 +574,10 @@ func updateRoots(ctx context.Context, direct map[string]bool, rs *Requirements, // To ensure that the loading process eventually converges, the caller should // add any needed roots from the tidy root set (without removing existing untidy // roots) until the set of roots has converged. -func tidyLazyRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) (*Requirements, error) { +func tidyLazyRoots(ctx context.Context, mainModule module.Version, direct map[string]bool, pkgs []*loadPkg) (*Requirements, error) { var ( roots []module.Version - pathIncluded = map[string]bool{Target.Path: true} + pathIncluded = map[string]bool{mainModule.Path: true} ) // We start by adding roots for every package in "all". // @@ -842,7 +855,9 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen roots = make([]module.Version, 0, len(rs.rootModules)) rootsUpgraded = false inRootPaths := make(map[string]bool, len(rs.rootModules)+1) - inRootPaths[Target.Path] = true + for _, mm := range MainModules.Versions() { + inRootPaths[mm.Path] = true + } for _, m := range rs.rootModules { if inRootPaths[m.Path] { // This root specifies a redundant path. We already retained the @@ -939,7 +954,7 @@ func spotCheckRoots(ctx context.Context, rs *Requirements, mods map[module.Versi // tidyEagerRoots returns a minimal set of root requirements that maintains the // selected version of every module that provided a package in pkgs, and // includes the selected version of every such module in direct as a root. -func tidyEagerRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg) (*Requirements, error) { +func tidyEagerRoots(ctx context.Context, mainModule module.Version, direct map[string]bool, pkgs []*loadPkg) (*Requirements, error) { var ( keep []module.Version keptPath = map[string]bool{} @@ -962,7 +977,7 @@ func tidyEagerRoots(ctx context.Context, direct map[string]bool, pkgs []*loadPkg } } - min, err := mvs.Req(Target, rootPaths, &mvsReqs{roots: keep}) + min, err := mvs.Req(mainModule, rootPaths, &mvsReqs{roots: keep}) if err != nil { return nil, err } @@ -1051,7 +1066,7 @@ func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requireme // This is only for convenience and clarity for end users: in an eager module, // the choice of explicit vs. implicit dependency has no impact on MVS // selection (for itself or any other module). - keep := append(mg.BuildList()[1:], add...) + keep := append(mg.BuildList()[MainModules.Len():], add...) for _, m := range keep { if direct[m.Path] && !inRootPaths[m.Path] { rootPaths = append(rootPaths, m.Path) @@ -1059,16 +1074,22 @@ func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requireme } } - min, err := mvs.Req(Target, rootPaths, &mvsReqs{roots: keep}) - if err != nil { - return rs, err + // TODO(matloob): Make roots into a map. + var roots []module.Version + for _, mainModule := range MainModules.Versions() { + min, err := mvs.Req(mainModule, rootPaths, &mvsReqs{roots: keep}) + if err != nil { + return rs, err + } + roots = append(roots, min...) } - if rs.depth == eager && reflect.DeepEqual(min, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { + if rs.depth == eager && reflect.DeepEqual(roots, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { // The root set is unchanged and rs was already eager, so keep rs to // preserve its cached ModuleGraph (if any). return rs, nil } - return newRequirements(eager, min, direct), nil + + return newRequirements(eager, roots, direct), nil } // convertDepth returns a version of rs with the given depth. @@ -1098,5 +1119,5 @@ func convertDepth(ctx context.Context, rs *Requirements, depth modDepth) (*Requi if err != nil { return rs, err } - return newRequirements(lazy, mg.BuildList()[1:], rs.direct), nil + return newRequirements(lazy, mg.BuildList()[MainModules.Len():], rs.direct), nil } diff --git a/src/cmd/go/internal/modload/edit.go b/src/cmd/go/internal/modload/edit.go index c350b9d1b5c..796721c90cf 100644 --- a/src/cmd/go/internal/modload/edit.go +++ b/src/cmd/go/internal/modload/edit.go @@ -75,7 +75,7 @@ func editRequirements(ctx context.Context, rs *Requirements, tryUpgrade, mustSel // We promote the modules in mustSelect to be explicit requirements. var rootPaths []string for _, m := range mustSelect { - if m.Version != "none" && m.Path != Target.Path { + if !MainModules.Contains(m.Path) && m.Version != "none" { rootPaths = append(rootPaths, m.Path) } } @@ -97,7 +97,7 @@ func editRequirements(ctx context.Context, rs *Requirements, tryUpgrade, mustSel } } - roots, err = mvs.Req(Target, rootPaths, &mvsReqs{roots: mods}) + roots, err = mvs.Req(MainModules.mustGetSingleMainModule(), rootPaths, &mvsReqs{roots: mods}) if err != nil { return nil, false, err } @@ -218,8 +218,8 @@ func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, d eagerUpgrades = tryUpgrade } else { for _, m := range tryUpgrade { - if m.Path == Target.Path { - // Target is already considered to be higher than any possible m, so we + if MainModules.Contains(m.Path) { + // The main module versions are already considered to be higher than any possible m, so we // won't be upgrading to it anyway and there is no point scanning its // dependencies. continue @@ -318,7 +318,7 @@ func selectPotentiallyImportedModules(ctx context.Context, limiter *versionLimit mods = make([]module.Version, 0, len(limiter.selected)) for path, v := range limiter.selected { - if v != "none" && path != Target.Path { + if v != "none" && !MainModules.Contains(path) { mods = append(mods, module.Version{Path: path, Version: v}) } } @@ -334,7 +334,7 @@ func selectPotentiallyImportedModules(ctx context.Context, limiter *versionLimit } mods = make([]module.Version, 0, len(limiter.selected)) for path, _ := range limiter.selected { - if path != Target.Path { + if !MainModules.Contains(path) { if v := mg.Selected(path); v != "none" { mods = append(mods, module.Version{Path: path, Version: v}) } @@ -415,10 +415,14 @@ func (dq dqState) isDisqualified() bool { // itself lazy, its unrestricted dependencies are skipped when scanning // requirements. func newVersionLimiter(depth modDepth, max map[string]string) *versionLimiter { + selected := make(map[string]string) + for _, m := range MainModules.Versions() { + selected[m.Path] = m.Version + } return &versionLimiter{ depth: depth, max: max, - selected: map[string]string{Target.Path: Target.Version}, + selected: selected, dqReason: map[module.Version]dqState{}, requiring: map[module.Version][]module.Version{}, } @@ -492,7 +496,7 @@ func (l *versionLimiter) Select(m module.Version) (conflict module.Version, err // as is feasible, we don't want to retain test dependencies that are only // marginally relevant at best. func (l *versionLimiter) check(m module.Version, depth modDepth) dqState { - if m.Version == "none" || m == Target { + if m.Version == "none" || m == MainModules.mustGetSingleMainModule() { // version "none" has no requirements, and the dependencies of Target are // tautological. return dqState{} diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index d2bbe5cbe0b..7b5305e4bb2 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -257,11 +257,13 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M // Is the package in the standard library? pathIsStd := search.IsStandardImportPath(path) if pathIsStd && goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) { - if targetInGorootSrc { - if dir, ok, err := dirInModule(path, targetPrefix, ModRoot(), true); err != nil { - return module.Version{}, dir, err - } else if ok { - return Target, dir, nil + for _, mainModule := range MainModules.Versions() { + if MainModules.InGorootSrc(mainModule) { + if dir, ok, err := dirInModule(path, MainModules.PathPrefix(mainModule), MainModules.ModRoot(mainModule), true); err != nil { + return module.Version{}, dir, err + } else if ok { + return mainModule, dir, nil + } } } dir := filepath.Join(cfg.GOROOT, "src", path) @@ -271,8 +273,10 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M // -mod=vendor is special. // Everything must be in the main module or the main module's vendor directory. if cfg.BuildMod == "vendor" { - mainDir, mainOK, mainErr := dirInModule(path, targetPrefix, ModRoot(), true) - vendorDir, vendorOK, _ := dirInModule(path, "", filepath.Join(ModRoot(), "vendor"), false) + mainModule := MainModules.mustGetSingleMainModule() + modRoot := MainModules.ModRoot(mainModule) + mainDir, mainOK, mainErr := dirInModule(path, MainModules.PathPrefix(mainModule), modRoot, true) + vendorDir, vendorOK, _ := dirInModule(path, "", filepath.Join(modRoot, "vendor"), false) if mainOK && vendorOK { return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: []string{mainDir, vendorDir}} } @@ -280,7 +284,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M // Note that we're not checking that the package exists. // We'll leave that for load. if !vendorOK && mainDir != "" { - return Target, mainDir, nil + return mainModule, mainDir, nil } if mainErr != nil { return module.Version{}, "", mainErr @@ -638,8 +642,8 @@ func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFile // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) { - if mod == Target { - return ModRoot(), true, nil + if modRoot := MainModules.ModRoot(mod); modRoot != "" { + return modRoot, true, nil } if r := Replacement(mod); r.Path != "" { if r.Version == "" { diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index cbc7289afa5..33f91630385 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -45,26 +45,108 @@ var ( allowMissingModuleImports bool ) +func TODOWorkspaces(s string) error { + return fmt.Errorf("need to support this for workspaces: %s", s) +} + // Variables set in Init. var ( initialized bool - modRoot string - gopath string + + // The directory containing go.work file. Set if in a go.work file is found + // and the go command is operating in workspace mode. + workRoot string + + // These are primarily used to initialize the MainModules, and should be + // eventually superceded by them but are still used in cases where the module + // roots are required but MainModules hasn't been initialized yet. Set to + // the modRoots of the main modules. + // modRoots != nil implies len(modRoots) > 0 + modRoots []string + gopath string ) -// Variables set in initTarget (during {Load,Create}ModFile). -var ( - Target module.Version +type MainModuleSet struct { + // versions are the module.Version values of each of the main modules. + // For each of them, the Path fields are ordinary module paths and the Version + // fields are empty strings. + versions []module.Version - // targetPrefix is the path prefix for packages in Target, without a trailing - // slash. For most modules, targetPrefix is just Target.Path, but the + // modRoot maps each module in versions to its absolute filesystem path. + modRoot map[module.Version]string + + // pathPrefix is the path prefix for packages in the module, without a trailing + // slash. For most modules, pathPrefix is just version.Path, but the // standard-library module "std" has an empty prefix. - targetPrefix string + pathPrefix map[module.Version]string - // targetInGorootSrc caches whether modRoot is within GOROOT/src. + // inGorootSrc caches whether modRoot is within GOROOT/src. // The "std" module is special within GOROOT/src, but not otherwise. - targetInGorootSrc bool -) + inGorootSrc map[module.Version]bool +} + +func (mms *MainModuleSet) PathPrefix(m module.Version) string { + return mms.pathPrefix[m] +} + +// Versions returns the module.Version values of each of the main modules. +// For each of them, the Path fields are ordinary module paths and the Version +// fields are empty strings. +// Callers should not modify the returned slice. +func (mms *MainModuleSet) Versions() []module.Version { + if mms == nil { + return nil + } + return mms.versions +} + +func (mms *MainModuleSet) Contains(path string) bool { + if mms == nil { + return false + } + for _, v := range mms.versions { + if v.Path == path { + return true + } + } + return false +} + +func (mms *MainModuleSet) ModRoot(m module.Version) string { + _ = TODOWorkspaces(" Do we need the Init? The original modRoot calls it. Audit callers.") + Init() + if mms == nil { + return "" + } + return mms.modRoot[m] +} + +func (mms *MainModuleSet) InGorootSrc(m module.Version) bool { + if mms == nil { + return false + } + return mms.inGorootSrc[m] +} + +func (mms *MainModuleSet) mustGetSingleMainModule() module.Version { + if mms == nil || len(mms.versions) == 0 { + panic("internal error: mustGetSingleMainModule called in context with no main modules") + } + if len(mms.versions) != 1 { + _ = TODOWorkspaces("Check if we're in workspace mode before returning the below error.") + panic("internal error: mustGetSingleMainModule called in workspace mode") + } + return mms.versions[0] +} + +func (mms *MainModuleSet) Len() int { + if mms == nil { + return 0 + } + return len(mms.versions) +} + +var MainModules *MainModuleSet type Root int @@ -169,18 +251,17 @@ func Init() { if os.Getenv("GCM_INTERACTIVE") == "" { os.Setenv("GCM_INTERACTIVE", "never") } - - if modRoot != "" { + if modRoots != nil { // modRoot set before Init was called ("go mod init" does this). // No need to search for go.mod. } else if RootMode == NoRoot { if cfg.ModFile != "" && !base.InGOFLAGS("-modfile") { base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") } - modRoot = "" + modRoots = nil } else { - modRoot = findModuleRoot(base.Cwd()) - if modRoot == "" { + modRoots = findModuleRoots(base.Cwd()) + if modRoots == nil { if cfg.ModFile != "" { base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") } @@ -192,13 +273,13 @@ func Init() { // Stay in GOPATH mode. return } - } else if search.InDir(modRoot, os.TempDir()) == "." { + } else if search.InDir(modRoots[0], os.TempDir()) == "." { // If you create /tmp/go.mod for experimenting, // then any tests that create work directories under /tmp // will find it and get modules when they're not expecting them. // It's a bit of a peculiar thing to disallow but quite mysterious // when it happens. See golang.org/issue/26708. - modRoot = "" + modRoots = nil fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) if !mustUseModules { return @@ -221,7 +302,7 @@ func Init() { base.Fatalf("$GOPATH/go.mod exists but should not") } - if modRoot == "" { + if modRoots == nil { // We're in module mode, but not inside a module. // // Commands like 'go build', 'go run', 'go list' have no go.mod file to @@ -240,8 +321,8 @@ func Init() { // // See golang.org/issue/32027. } else { + _ = TODOWorkspaces("Instead of modfile path, find modfile OR workfile path depending on mode") modfetch.GoSumFile = strings.TrimSuffix(ModFilePath(), ".mod") + ".sum" - search.SetModRoot(modRoot) } } @@ -255,7 +336,7 @@ func Init() { // be called until the command is installed and flags are parsed. Instead of // calling Init and Enabled, the main package can call this function. func WillBeEnabled() bool { - if modRoot != "" || cfg.ModulesEnabled { + if modRoots != nil || cfg.ModulesEnabled { // Already enabled. return true } @@ -276,11 +357,12 @@ func WillBeEnabled() bool { return false } - if modRoot := findModuleRoot(base.Cwd()); modRoot == "" { + if modRoots := findModuleRoots(base.Cwd()); modRoots == nil { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. return false - } else if search.InDir(modRoot, os.TempDir()) == "." { + } else if search.InDir(modRoots[0], os.TempDir()) == "." { + _ = TODOWorkspaces("modRoots[0] is not right here") // If you create /tmp/go.mod for experimenting, // then any tests that create work directories under /tmp // will find it and get modules when they're not expecting them. @@ -297,7 +379,7 @@ func WillBeEnabled() bool { // (usually through MustModRoot). func Enabled() bool { Init() - return modRoot != "" || cfg.ModulesEnabled + return modRoots != nil || cfg.ModulesEnabled } // ModRoot returns the root of the main module. @@ -306,7 +388,10 @@ func ModRoot() string { if !HasModRoot() { die() } - return modRoot + if len(modRoots) != 1 { + panic(TODOWorkspaces("need to handle multiple modroots here")) + } + return modRoots[0] } // HasModRoot reports whether a main module is present. @@ -314,7 +399,7 @@ func ModRoot() string { // does not require a main module. func HasModRoot() bool { Init() - return modRoot != "" + return modRoots != nil } // ModFilePath returns the effective path of the go.mod file. Normally, this @@ -322,9 +407,10 @@ func HasModRoot() bool { // change its location. ModFilePath calls base.Fatalf if there is no main // module, even if -modfile is set. func ModFilePath() string { - if !HasModRoot() { - die() - } + return modFilePath(ModRoot()) +} + +func modFilePath(modRoot string) string { if cfg.ModFile != "" { return cfg.ModFile } @@ -402,11 +488,12 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { } Init() - if modRoot == "" { - Target = module.Version{Path: "command-line-arguments"} - targetPrefix = "command-line-arguments" + if len(modRoots) == 0 { + _ = TODOWorkspaces("Instead of creating a fake module with an empty modroot, make MainModules.Len() == 0 mean that we're in module mode but not inside any module.") + mainModule := module.Version{Path: "command-line-arguments"} + MainModules = makeMainModules([]module.Version{mainModule}, []string{""}) goVersion := LatestGoVersion() - rawGoVersion.Store(Target, goVersion) + rawGoVersion.Store(mainModule, goVersion) requirements = newRequirements(modDepthFromGoVersion(goVersion), nil, nil) return requirements, false } @@ -428,9 +515,13 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod") } + // For now, this code assumes there's a single main module, because there's + // no way to specify multiple main modules yet. TODO(#45713): update this + // in a later CL. modFile = f - initTarget(f.Module.Mod) - index = indexModFile(data, f, fixed) + mainModule := f.Module.Mod + MainModules = makeMainModules([]module.Version{mainModule}, modRoots) + index = indexModFile(data, f, mainModule, fixed) if err := module.CheckImportPath(f.Module.Mod.Path); err != nil { if pathErr, ok := err.(*module.InvalidPathError); ok { @@ -451,7 +542,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { // TODO(#45551): Do something more principled instead of checking // cfg.CmdName directly here. if cfg.BuildMod == "mod" && cfg.CmdName != "mod graph" && cfg.CmdName != "mod why" { - addGoStmt(LatestGoVersion()) + addGoStmt(mainModule, LatestGoVersion()) if go117EnableLazyLoading { // We need to add a 'go' version to the go.mod file, but we must assume // that its existing contents match something between Go 1.11 and 1.16. @@ -464,7 +555,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { } } } else { - rawGoVersion.Store(Target, modFileGoVersion()) + rawGoVersion.Store(mainModule, modFileGoVersion()) } } @@ -482,7 +573,8 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { // exactly the same as in the legacy configuration (for example, we can't get // packages at multiple versions from the same module). func CreateModFile(ctx context.Context, modPath string) { - modRoot = base.Cwd() + modRoot := base.Cwd() + modRoots = []string{modRoot} Init() modFilePath := ModFilePath() if _, err := fsys.Stat(modFilePath); err == nil { @@ -510,8 +602,8 @@ func CreateModFile(ctx context.Context, modPath string) { fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) modFile = new(modfile.File) modFile.AddModuleStmt(modPath) - initTarget(modFile.Module.Mod) - addGoStmt(LatestGoVersion()) // Add the go directive before converted module requirements. + MainModules = makeMainModules([]module.Version{modFile.Module.Mod}, []string{modRoot}) + addGoStmt(modFile.Module.Mod, LatestGoVersion()) // Add the go directive before converted module requirements. convertedFrom, err := convertLegacyConfig(modPath) if convertedFrom != "" { @@ -609,32 +701,50 @@ func AllowMissingModuleImports() { allowMissingModuleImports = true } -// initTarget sets Target and associated variables according to modFile, -func initTarget(m module.Version) { - Target = m - targetPrefix = m.Path - - if rel := search.InDir(base.Cwd(), cfg.GOROOTsrc); rel != "" { - targetInGorootSrc = true - if m.Path == "std" { - // The "std" module in GOROOT/src is the Go standard library. Unlike other - // modules, the packages in the "std" module have no import-path prefix. - // - // Modules named "std" outside of GOROOT/src do not receive this special - // treatment, so it is possible to run 'go test .' in other GOROOTs to - // test individual packages using a combination of the modified package - // and the ordinary standard library. - // (See https://golang.org/issue/30756.) - targetPrefix = "" +// makeMainModules creates a MainModuleSet and associated variables according to +// the given main modules. +func makeMainModules(ms []module.Version, rootDirs []string) *MainModuleSet { + for _, m := range ms { + if m.Version != "" { + panic("mainModulesCalled with module.Version with non empty Version field: " + fmt.Sprintf("%#v", m)) } } + mainModules := &MainModuleSet{ + versions: ms[:len(ms):len(ms)], + inGorootSrc: map[module.Version]bool{}, + pathPrefix: map[module.Version]string{}, + modRoot: map[module.Version]string{}, + } + for i, m := range ms { + mainModules.pathPrefix[m] = m.Path + mainModules.modRoot[m] = rootDirs[i] + + if rel := search.InDir(rootDirs[i], cfg.GOROOTsrc); rel != "" { + mainModules.inGorootSrc[m] = true + if m.Path == "std" { + // The "std" module in GOROOT/src is the Go standard library. Unlike other + // modules, the packages in the "std" module have no import-path prefix. + // + // Modules named "std" outside of GOROOT/src do not receive this special + // treatment, so it is possible to run 'go test .' in other GOROOTs to + // test individual packages using a combination of the modified package + // and the ordinary standard library. + // (See https://golang.org/issue/30756.) + mainModules.pathPrefix[m] = "" + } + } + } + return mainModules } // requirementsFromModFile returns the set of non-excluded requirements from // the global modFile. func requirementsFromModFile(ctx context.Context) *Requirements { roots := make([]module.Version, 0, len(modFile.Require)) - mPathCount := map[string]int{Target.Path: 1} + mPathCount := make(map[string]int) + for _, m := range MainModules.Versions() { + mPathCount[m.Path] = 1 + } direct := map[string]bool{} for _, r := range modFile.Require { if index != nil && index.exclude[r.Mod] { @@ -687,7 +797,7 @@ func setDefaultBuildMod() { cfg.BuildMod = "mod" return } - if modRoot == "" { + if modRoots == nil { if allowMissingModuleImports { cfg.BuildMod = "mod" } else { @@ -696,23 +806,25 @@ func setDefaultBuildMod() { return } - if fi, err := fsys.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() { - modGo := "unspecified" - if index != nil && index.goVersionV != "" { - if semver.Compare(index.goVersionV, "v1.14") >= 0 { - // The Go version is at least 1.14, and a vendor directory exists. - // Set -mod=vendor by default. - cfg.BuildMod = "vendor" - cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." - return - } else { - modGo = index.goVersionV[1:] + if len(modRoots) == 1 { + if fi, err := fsys.Stat(filepath.Join(modRoots[0], "vendor")); err == nil && fi.IsDir() { + modGo := "unspecified" + if index != nil && index.goVersionV != "" { + if semver.Compare(index.goVersionV, "v1.14") >= 0 { + // The Go version is at least 1.14, and a vendor directory exists. + // Set -mod=vendor by default. + cfg.BuildMod = "vendor" + cfg.BuildModReason = "Go version in go.mod is at least 1.14 and vendor directory exists." + return + } else { + modGo = index.goVersionV[1:] + } } - } - // Since a vendor directory exists, we should record why we didn't use it. - // This message won't normally be shown, but it may appear with import errors. - cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) + // Since a vendor directory exists, we should record why we didn't use it. + // This message won't normally be shown, but it may appear with import errors. + cfg.BuildModReason = fmt.Sprintf("Go version in go.mod is %s, so vendor directory was not used.", modGo) + } } cfg.BuildMod = "readonly" @@ -733,7 +845,10 @@ func convertLegacyConfig(modPath string) (from string, err error) { return modOnly.Mod, nil } for _, name := range altConfigs { - cfg := filepath.Join(modRoot, name) + if len(modRoots) != 1 { + panic(TODOWorkspaces("what do do here?")) + } + cfg := filepath.Join(modRoots[0], name) data, err := os.ReadFile(cfg) if err == nil { convert := modconv.Converters[name] @@ -751,14 +866,14 @@ func convertLegacyConfig(modPath string) (from string, err error) { // addGoStmt adds a go directive to the go.mod file if it does not already // include one. The 'go' version added, if any, is the latest version supported // by this toolchain. -func addGoStmt(v string) { +func addGoStmt(mod module.Version, v string) { if modFile.Go != nil && modFile.Go.Version != "" { return } if err := modFile.AddGoStmt(v); err != nil { base.Fatalf("go: internal error: %v", err) } - rawGoVersion.Store(Target, v) + rawGoVersion.Store(mod, v) } // LatestGoVersion returns the latest version of the Go language supported by @@ -809,7 +924,7 @@ var altConfigs = []string{ ".git/config", } -func findModuleRoot(dir string) (root string) { +func findModuleRoots(dir string) (roots []string) { if dir == "" { panic("dir not set") } @@ -818,7 +933,7 @@ func findModuleRoot(dir string) (root string) { // Look for enclosing go.mod. for { if fi, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { - return dir + return []string{dir} } d := filepath.Dir(dir) if d == dir { @@ -826,7 +941,7 @@ func findModuleRoot(dir string) (root string) { } dir = d } - return "" + return nil } func findAltConfig(dir string) (root, name string) { @@ -987,7 +1102,8 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) return } - if modRoot == "" { + if MainModules.Len() != 1 || MainModules.ModRoot(MainModules.Versions()[0]) == "" { + _ = TODOWorkspaces("also check that workspace mode is off") // We aren't in a module, so we don't have anywhere to write a go.mod file. return } @@ -1032,8 +1148,14 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) base.Fatalf("go: %v", err) } defer func() { + if MainModules.Len() != 1 { + panic(TODOWorkspaces("There should be exactly one main module when committing reqs")) + } + + mainModule := MainModules.Versions()[0] + // At this point we have determined to make the go.mod file on disk equal to new. - index = indexModFile(new, modFile, false) + index = indexModFile(new, modFile, mainModule, false) // Update go.sum after releasing the side lock and refreshing the index. // 'go mod init' shouldn't write go.sum, since it will be incomplete. diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index ccdeb9b1d11..1862bef4945 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -79,7 +79,11 @@ func ListModules(ctx context.Context, args []string, mode ListMode) ([]*modinfo. func listModules(ctx context.Context, rs *Requirements, args []string, mode ListMode) (_ *Requirements, mods []*modinfo.ModulePublic, mgErr error) { if len(args) == 0 { - return rs, []*modinfo.ModulePublic{moduleInfo(ctx, rs, Target, mode)}, nil + var ms []*modinfo.ModulePublic + for _, m := range MainModules.Versions() { + ms = append(ms, moduleInfo(ctx, rs, m, mode)) + } + return rs, ms, nil } needFullGraph := false diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index a3a8021c049..77d2dc40303 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -274,7 +274,9 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma // If we're outside of a module, ensure that the failure mode // indicates that. - ModRoot() + if !HasModRoot() { + die() + } if ld != nil { m.AddError(err) @@ -306,7 +308,7 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma // The initial roots are the packages in the main module. // loadFromRoots will expand that to "all". m.Errs = m.Errs[:0] - matchPackages(ctx, m, opts.Tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, opts.Tags, omitStd, MainModules.Versions()) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -443,7 +445,7 @@ func matchLocalDirs(ctx context.Context, m *search.Match, rs *Requirements) { } } - m.MatchDirs() + m.MatchDirs(modRoots) } // resolveLocalPackage resolves a filesystem path to a package path. @@ -485,49 +487,70 @@ func resolveLocalPackage(ctx context.Context, dir string, rs *Requirements) (str } } - if modRoot != "" && absDir == modRoot { - if absDir == cfg.GOROOTsrc { - return "", errPkgIsGorootSrc + for _, mod := range MainModules.Versions() { + modRoot := MainModules.ModRoot(mod) + if modRoot != "" && absDir == modRoot { + if absDir == cfg.GOROOTsrc { + return "", errPkgIsGorootSrc + } + return MainModules.PathPrefix(mod), nil } - return targetPrefix, nil } // Note: The checks for @ here are just to avoid misinterpreting // the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar). // It's not strictly necessary but helpful to keep the checks. - if modRoot != "" && strings.HasPrefix(absDir, modRoot+string(filepath.Separator)) && !strings.Contains(absDir[len(modRoot):], "@") { - suffix := filepath.ToSlash(absDir[len(modRoot):]) - if strings.HasPrefix(suffix, "/vendor/") { - if cfg.BuildMod != "vendor" { - return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) + var pkgNotFoundErr error + pkgNotFoundLongestPrefix := "" + for _, mainModule := range MainModules.Versions() { + modRoot := MainModules.ModRoot(mainModule) + if modRoot != "" && strings.HasPrefix(absDir, modRoot+string(filepath.Separator)) && !strings.Contains(absDir[len(modRoot):], "@") { + suffix := filepath.ToSlash(absDir[len(modRoot):]) + if strings.HasPrefix(suffix, "/vendor/") { + if cfg.BuildMod != "vendor" { + return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) + } + + readVendorList() + pkg := strings.TrimPrefix(suffix, "/vendor/") + if _, ok := vendorPkgModule[pkg]; !ok { + return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) + } + return pkg, nil } - readVendorList() - pkg := strings.TrimPrefix(suffix, "/vendor/") - if _, ok := vendorPkgModule[pkg]; !ok { - return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) + mainModulePrefix := MainModules.PathPrefix(mainModule) + if mainModulePrefix == "" { + pkg := strings.TrimPrefix(suffix, "/") + if pkg == "builtin" { + // "builtin" is a pseudo-package with a real source file. + // It's not included in "std", so it shouldn't resolve from "." + // within module "std" either. + return "", errPkgIsBuiltin + } + return pkg, nil + } + + pkg := mainModulePrefix + suffix + if _, ok, err := dirInModule(pkg, mainModulePrefix, modRoot, true); err != nil { + return "", err + } else if !ok { + // This main module could contain the directory but doesn't. Other main + // modules might contain the directory, so wait till we finish the loop + // to see if another main module contains directory. But if not, + // return an error. + if len(mainModulePrefix) > len(pkgNotFoundLongestPrefix) { + pkgNotFoundLongestPrefix = mainModulePrefix + pkgNotFoundErr = &PackageNotInModuleError{Mod: mainModule, Pattern: pkg} + + } + continue } return pkg, nil } - - if targetPrefix == "" { - pkg := strings.TrimPrefix(suffix, "/") - if pkg == "builtin" { - // "builtin" is a pseudo-package with a real source file. - // It's not included in "std", so it shouldn't resolve from "." - // within module "std" either. - return "", errPkgIsBuiltin - } - return pkg, nil - } - - pkg := targetPrefix + suffix - if _, ok, err := dirInModule(pkg, targetPrefix, modRoot, true); err != nil { - return "", err - } else if !ok { - return "", &PackageNotInModuleError{Mod: Target, Pattern: pkg} - } - return pkg, nil + } + if pkgNotFoundErr != nil { + return "", pkgNotFoundErr } if sub := search.InDir(absDir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") { @@ -649,10 +672,10 @@ func ImportFromFiles(ctx context.Context, gofiles []string) { } // DirImportPath returns the effective import path for dir, -// provided it is within the main module, or else returns ".". -func DirImportPath(ctx context.Context, dir string) string { +// provided it is within a main module, or else returns ".". +func (mms *MainModuleSet) DirImportPath(ctx context.Context, dir string) (path string, m module.Version) { if !HasModRoot() { - return "." + return ".", module.Version{} } LoadModFile(ctx) // Sets targetPrefix. @@ -662,17 +685,32 @@ func DirImportPath(ctx context.Context, dir string) string { dir = filepath.Clean(dir) } - if dir == modRoot { - return targetPrefix - } - if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) { - suffix := filepath.ToSlash(dir[len(modRoot):]) - if strings.HasPrefix(suffix, "/vendor/") { - return strings.TrimPrefix(suffix, "/vendor/") + var longestPrefix string + var longestPrefixPath string + var longestPrefixVersion module.Version + for _, v := range mms.Versions() { + modRoot := mms.ModRoot(v) + if dir == modRoot { + return mms.PathPrefix(v), v + } + if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) { + pathPrefix := MainModules.PathPrefix(v) + if pathPrefix > longestPrefix { + longestPrefix = pathPrefix + longestPrefixVersion = v + suffix := filepath.ToSlash(dir[len(modRoot):]) + if strings.HasPrefix(suffix, "/vendor/") { + longestPrefixPath = strings.TrimPrefix(suffix, "/vendor/") + } + longestPrefixPath = mms.PathPrefix(v) + suffix + } } - return targetPrefix + suffix } - return "." + if len(longestPrefix) > 0 { + return longestPrefixPath, longestPrefixVersion + } + + return ".", module.Version{} } // TargetPackages returns the list of packages in the target (top-level) module @@ -685,7 +723,7 @@ func TargetPackages(ctx context.Context, pattern string) *search.Match { ModRoot() // Emits an error if Target cannot contain packages. m := search.NewMatch(pattern) - matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, MainModules.Versions()) return m } @@ -931,10 +969,7 @@ func (pkg *loadPkg) fromExternalModule() bool { if pkg.mod.Path == "" { return false // loaded from the standard library, not a module } - if pkg.mod.Path == Target.Path { - return false // loaded from the main module. - } - return true + return !MainModules.Contains(pkg.mod.Path) } var errMissing = errors.New("cannot find package") @@ -1205,7 +1240,7 @@ func (ld *loader) updateRequirements(ctx context.Context) (changed bool, err err } for _, pkg := range ld.pkgs { - if pkg.mod != Target { + if pkg.mod.Version != "" || !MainModules.Contains(pkg.mod.Path) { continue } for _, dep := range pkg.imports { @@ -1462,7 +1497,7 @@ func (ld *loader) applyPkgFlags(ctx context.Context, pkg *loadPkg, flags loadPkg // so it's ok if we call it more than is strictly necessary. wantTest := false switch { - case ld.allPatternIsRoot && pkg.mod == Target: + case ld.allPatternIsRoot && MainModules.Contains(pkg.mod.Path): // We are loading the "all" pattern, which includes packages imported by // tests in the main module. This package is in the main module, so we // need to identify the imports of its test even if LoadTests is not set. @@ -1483,7 +1518,7 @@ func (ld *loader) applyPkgFlags(ctx context.Context, pkg *loadPkg, flags loadPkg if wantTest { var testFlags loadPkgFlags - if pkg.mod == Target || (ld.allClosesOverTests && new.has(pkgInAll)) { + if MainModules.Contains(pkg.mod.Path) || (ld.allClosesOverTests && new.has(pkgInAll)) { // Tests of packages in the main module are in "all", in the sense that // they cause the packages they import to also be in "all". So are tests // of packages in "all" if "all" closes over test dependencies. @@ -1630,7 +1665,7 @@ func (ld *loader) load(ctx context.Context, pkg *loadPkg) { if pkg.dir == "" { return } - if pkg.mod == Target { + if MainModules.Contains(pkg.mod.Path) { // Go ahead and mark pkg as in "all". This provides the invariant that a // package that is *only* imported by other packages in "all" is always // marked as such before loading its imports. @@ -1735,13 +1770,14 @@ func (ld *loader) stdVendor(parentPath, path string) string { } if str.HasPathPrefix(parentPath, "cmd") { - if !ld.VendorModulesInGOROOTSrc || Target.Path != "cmd" { + if !ld.VendorModulesInGOROOTSrc || !MainModules.Contains("cmd") { vendorPath := pathpkg.Join("cmd", "vendor", path) + if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil { return vendorPath } } - } else if !ld.VendorModulesInGOROOTSrc || Target.Path != "std" || str.HasPathPrefix(parentPath, "vendor") { + } else if !ld.VendorModulesInGOROOTSrc || !MainModules.Contains("std") || str.HasPathPrefix(parentPath, "vendor") { // If we are outside of the 'std' module, resolve imports from within 'std' // to the vendor directory. // diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 1145ac4ba59..7b9f6e863aa 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -331,7 +331,7 @@ func resolveReplacement(m module.Version) module.Version { // indexModFile rebuilds the index of modFile. // If modFile has been changed since it was first read, // modFile.Cleanup must be called before indexModFile. -func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileIndex { +func indexModFile(data []byte, modFile *modfile.File, mod module.Version, needsFix bool) *modFileIndex { i := new(modFileIndex) i.data = data i.dataNeedsFix = needsFix @@ -343,12 +343,12 @@ func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileInd i.goVersionV = "" if modFile.Go == nil { - rawGoVersion.Store(Target, "") + rawGoVersion.Store(mod, "") } else { // We're going to use the semver package to compare Go versions, so go ahead // and add the "v" prefix it expects once instead of every time. i.goVersionV = "v" + modFile.Go.Version - rawGoVersion.Store(Target, modFile.Go.Version) + rawGoVersion.Store(mod, modFile.Go.Version) } i.require = make(map[module.Version]requireMeta, len(modFile.Require)) @@ -488,8 +488,8 @@ type retraction struct { // // The caller must not modify the returned summary. func goModSummary(m module.Version) (*modFileSummary, error) { - if m == Target { - panic("internal error: goModSummary called on the Target module") + if m.Version == "" && MainModules.Contains(m.Path) { + panic("internal error: goModSummary called on a main module") } if cfg.BuildMod == "vendor" { @@ -583,7 +583,7 @@ func goModSummary(m module.Version) (*modFileSummary, error) { // // rawGoModSummary cannot be used on the Target module. func rawGoModSummary(m module.Version) (*modFileSummary, error) { - if m == Target { + if m.Path == "" && MainModules.Contains(m.Path) { panic("internal error: rawGoModSummary called on the Target module") } diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 87619b4ace6..40224d534b1 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -42,7 +42,7 @@ type mvsReqs struct { } func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) { - if mod == Target { + if MainModules.Contains(mod.Path) { // Use the build list as it existed when r was constructed, not the current // global build list. return r.roots, nil @@ -113,7 +113,7 @@ func versions(ctx context.Context, path string, allowed AllowedFunc) ([]string, func previousVersion(m module.Version) (module.Version, error) { // TODO(golang.org/issue/38714): thread tracing context through MVS. - if m == Target { + if MainModules.Contains(m.Path) { return module.Version{Path: m.Path, Version: "none"}, nil } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 6f6c6e8c98d..83e80d009b8 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -110,11 +110,12 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed allowed = func(context.Context, module.Version) error { return nil } } - if path == Target.Path && (query == "upgrade" || query == "patch") { - if err := allowed(ctx, Target); err != nil { + if MainModules.Contains(path) && (query == "upgrade" || query == "patch") { + m := module.Version{Path: path} + if err := allowed(ctx, m); err != nil { return nil, fmt.Errorf("internal error: main module version is not allowed: %w", err) } - return &modfetch.RevInfo{Version: Target.Version}, nil + return &modfetch.RevInfo{Version: m.Version}, nil } if path == "std" || path == "cmd" { @@ -551,7 +552,7 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin return m.Errs[0] } - var match func(mod module.Version, root string, isLocal bool) *search.Match + var match func(mod module.Version, roots []string, isLocal bool) *search.Match matchPattern := search.MatchPattern(pattern) if i := strings.Index(pattern, "..."); i >= 0 { @@ -559,30 +560,32 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin if base == "." { return nil, nil, &WildcardInFirstElementError{Pattern: pattern, Query: query} } - match = func(mod module.Version, root string, isLocal bool) *search.Match { + match = func(mod module.Version, roots []string, isLocal bool) *search.Match { m := search.NewMatch(pattern) matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { - match = func(mod module.Version, root string, isLocal bool) *search.Match { + match = func(mod module.Version, roots []string, isLocal bool) *search.Match { m := search.NewMatch(pattern) prefix := mod.Path - if mod == Target { - prefix = targetPrefix + if MainModules.Contains(mod.Path) { + prefix = MainModules.PathPrefix(module.Version{Path: mod.Path}) } - if _, ok, err := dirInModule(pattern, prefix, root, isLocal); err != nil { - m.AddError(err) - } else if ok { - m.Pkgs = []string{pattern} + for _, root := range roots { + if _, ok, err := dirInModule(pattern, prefix, root, isLocal); err != nil { + m.AddError(err) + } else if ok { + m.Pkgs = []string{pattern} + } } return m } } - var queryMatchesMainModule bool - if HasModRoot() { - m := match(Target, modRoot, true) + var mainModuleMatches []module.Version + for _, mainModule := range MainModules.Versions() { + m := match(mainModule, modRoots, true) if len(m.Pkgs) > 0 { if query != "upgrade" && query != "patch" { return nil, nil, &QueryMatchesPackagesInMainModuleError{ @@ -591,12 +594,12 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin Packages: m.Pkgs, } } - if err := allowed(ctx, Target); err != nil { - return nil, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, Target.Path, err) + if err := allowed(ctx, mainModule); err != nil { + return nil, nil, fmt.Errorf("internal error: package %s is in the main module (%s), but version is not allowed: %w", pattern, mainModule.Path, err) } return []QueryResult{{ - Mod: Target, - Rev: &modfetch.RevInfo{Version: Target.Version}, + Mod: mainModule, + Rev: &modfetch.RevInfo{Version: mainModule.Version}, Packages: m.Pkgs, }}, nil, nil } @@ -604,15 +607,17 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin return nil, nil, err } - if matchPattern(Target.Path) { - queryMatchesMainModule = true + var matchesMainModule bool + if matchPattern(mainModule.Path) { + mainModuleMatches = append(mainModuleMatches, mainModule) + matchesMainModule = true } - if (query == "upgrade" || query == "patch") && queryMatchesMainModule { - if err := allowed(ctx, Target); err == nil { + if (query == "upgrade" || query == "patch") && matchesMainModule { + if err := allowed(ctx, mainModule); err == nil { modOnly = &QueryResult{ - Mod: Target, - Rev: &modfetch.RevInfo{Version: Target.Version}, + Mod: mainModule, + Rev: &modfetch.RevInfo{Version: mainModule.Version}, } } } @@ -625,14 +630,17 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin if len(candidateModules) == 0 { if modOnly != nil { return nil, modOnly, nil - } else if queryMatchesMainModule { + } else if len(mainModuleMatches) != 0 { + _ = TODOWorkspaces("add multiple main modules to the error?") return nil, nil, &QueryMatchesMainModuleError{ - Pattern: pattern, - Query: query, + MainModule: mainModuleMatches[0], + Pattern: pattern, + Query: query, } } else { + _ = TODOWorkspaces("This should maybe be PackageNotInModule*s* error with the main modules that are prefixes of base") return nil, nil, &PackageNotInModuleError{ - Mod: Target, + Mod: MainModules.Versions()[0], Query: query, Pattern: pattern, } @@ -656,7 +664,7 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin if err != nil { return r, err } - m := match(r.Mod, root, isLocal) + m := match(r.Mod, []string{root}, isLocal) r.Packages = m.Pkgs if len(r.Packages) == 0 && !matchPattern(path) { if err := firstError(m); err != nil { @@ -684,7 +692,7 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin return err }) - if queryMatchesMainModule && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { + if len(mainModuleMatches) > 0 && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { return nil, nil, &QueryMatchesMainModuleError{ Pattern: pattern, Query: query, @@ -701,8 +709,13 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin func modulePrefixesExcludingTarget(path string) []string { prefixes := make([]string, 0, strings.Count(path, "/")+1) + mainModulePrefixes := make(map[string]bool) + for _, m := range MainModules.Versions() { + mainModulePrefixes[m.Path] = true + } + for { - if path != targetPrefix { + if !mainModulePrefixes[path] { if _, _, ok := module.SplitPathVersion(path); ok { prefixes = append(prefixes, path) } @@ -759,7 +772,7 @@ func queryPrefixModules(ctx context.Context, candidateModules []string, queryMod case *PackageNotInModuleError: // Given the option, prefer to attribute “package not in module” // to modules other than the main one. - if noPackage == nil || noPackage.Mod == Target { + if noPackage == nil || MainModules.Contains(noPackage.Mod.Path) { noPackage = rErr } case *NoMatchingVersionError: @@ -885,11 +898,11 @@ type PackageNotInModuleError struct { } func (e *PackageNotInModuleError) Error() string { - if e.Mod == Target { + if MainModules.Contains(e.Mod.Path) { if strings.Contains(e.Pattern, "...") { - return fmt.Sprintf("main module (%s) does not contain packages matching %s", Target.Path, e.Pattern) + return fmt.Sprintf("main module (%s) does not contain packages matching %s", e.Mod.Path, e.Pattern) } - return fmt.Sprintf("main module (%s) does not contain package %s", Target.Path, e.Pattern) + return fmt.Sprintf("main module (%s) does not contain package %s", e.Mod.Path, e.Pattern) } found := "" @@ -1094,16 +1107,34 @@ func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) // a version of the main module that cannot be satisfied. // (The main module's version cannot be changed.) type QueryMatchesMainModuleError struct { - Pattern string - Query string + MainModule module.Version + Pattern string + Query string } func (e *QueryMatchesMainModuleError) Error() string { - if e.Pattern == Target.Path { + if MainModules.Contains(e.Pattern) { return fmt.Sprintf("can't request version %q of the main module (%s)", e.Query, e.Pattern) } - return fmt.Sprintf("can't request version %q of pattern %q that includes the main module (%s)", e.Query, e.Pattern, Target.Path) + return fmt.Sprintf("can't request version %q of pattern %q that includes the main module (%s)", e.Query, e.Pattern, e.MainModule.Path) +} + +// A QueryUpgradesAllError indicates that a query requests +// an upgrade on the all pattern. +// (The main module's version cannot be changed.) +type QueryUpgradesAllError struct { + MainModules []module.Version + Query string +} + +func (e *QueryUpgradesAllError) Error() string { + var plural string = "" + if len(e.MainModules) != 1 { + plural = "s" + } + + return fmt.Sprintf("can't request version %q of pattern \"all\" that includes the main module%s", e.Query, plural) } // A QueryMatchesPackagesInMainModuleError indicates that a query cannot be diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index 658fc6f55a9..799c48e50a8 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -131,9 +131,10 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f } if cfg.BuildMod == "vendor" { - if HasModRoot() { - walkPkgs(ModRoot(), targetPrefix, pruneGoMod|pruneVendor) - walkPkgs(filepath.Join(ModRoot(), "vendor"), "", pruneVendor) + mod := MainModules.mustGetSingleMainModule() + if modRoot := MainModules.ModRoot(mod); modRoot != "" { + walkPkgs(modRoot, MainModules.PathPrefix(mod), pruneGoMod|pruneVendor) + walkPkgs(filepath.Join(modRoot, "vendor"), "", pruneVendor) } return } @@ -147,12 +148,12 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f root, modPrefix string isLocal bool ) - if mod == Target { - if !HasModRoot() { + if MainModules.Contains(mod.Path) { + if MainModules.ModRoot(mod) == "" { continue // If there is no main module, we can't search in it. } - root = ModRoot() - modPrefix = targetPrefix + root = MainModules.ModRoot(mod) + modPrefix = MainModules.PathPrefix(mod) isLocal = true } else { var err error diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go index 80713b0812e..e26da15a8f9 100644 --- a/src/cmd/go/internal/modload/vendor.go +++ b/src/cmd/go/internal/modload/vendor.go @@ -219,6 +219,7 @@ func checkVendorConsistency() { } if vendErrors.Len() > 0 { + modRoot := MainModules.ModRoot(MainModules.mustGetSingleMainModule()) base.Fatalf("go: inconsistent vendoring in %s:%s\n\n\tTo ignore the vendor directory, use -mod=readonly or -mod=mod.\n\tTo sync the vendor directory, run:\n\t\tgo mod vendor", modRoot, vendErrors) } } diff --git a/src/cmd/go/internal/mvs/mvs.go b/src/cmd/go/internal/mvs/mvs.go index 6969f90f2e6..566fa4b6b39 100644 --- a/src/cmd/go/internal/mvs/mvs.go +++ b/src/cmd/go/internal/mvs/mvs.go @@ -8,6 +8,7 @@ package mvs import ( "fmt" + "reflect" "sort" "sync" @@ -85,11 +86,11 @@ type DowngradeReqs interface { // of the list are sorted by path. // // See https://research.swtch.com/vgo-mvs for details. -func BuildList(target module.Version, reqs Reqs) ([]module.Version, error) { - return buildList(target, reqs, nil) +func BuildList(targets []module.Version, reqs Reqs) ([]module.Version, error) { + return buildList(targets, reqs, nil) } -func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (module.Version, error)) ([]module.Version, error) { +func buildList(targets []module.Version, reqs Reqs, upgrade func(module.Version) (module.Version, error)) ([]module.Version, error) { cmp := func(v1, v2 string) int { if reqs.Max(v1, v2) != v1 { return -1 @@ -102,7 +103,7 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m var ( mu sync.Mutex - g = NewGraph(cmp, []module.Version{target}) + g = NewGraph(cmp, targets) upgrades = map[module.Version]module.Version{} errs = map[module.Version]error{} // (non-nil errors only) ) @@ -110,7 +111,9 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m // Explore work graph in parallel in case reqs.Required // does high-latency network operations. var work par.Work - work.Add(target) + for _, target := range targets { + work.Add(target) + } work.Do(10, func(item interface{}) { m := item.(module.Version) @@ -168,12 +171,12 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m // The final list is the minimum version of each module found in the graph. list := g.BuildList() - if v := list[0]; v != target { + if vs := list[:len(targets)]; !reflect.DeepEqual(vs, targets) { // target.Version will be "" for modload, the main client of MVS. // "" denotes the main module, which has no version. However, MVS treats // version strings as opaque, so "" is not a special value here. // See golang.org/issue/31491, golang.org/issue/29773. - panic(fmt.Sprintf("mistake: chose version %q instead of target %+v", v, target)) + panic(fmt.Sprintf("mistake: chose versions %+v instead of targets %+v", vs, targets)) } return list, nil } @@ -181,8 +184,8 @@ func buildList(target module.Version, reqs Reqs, upgrade func(module.Version) (m // Req returns the minimal requirement list for the target module, // with the constraint that all module paths listed in base must // appear in the returned list. -func Req(target module.Version, base []string, reqs Reqs) ([]module.Version, error) { - list, err := BuildList(target, reqs) +func Req(mainModule module.Version, base []string, reqs Reqs) ([]module.Version, error) { + list, err := BuildList([]module.Version{mainModule}, reqs) if err != nil { return nil, err } @@ -194,7 +197,8 @@ func Req(target module.Version, base []string, reqs Reqs) ([]module.Version, err // Compute postorder, cache requirements. var postorder []module.Version reqCache := map[module.Version][]module.Version{} - reqCache[target] = nil + reqCache[mainModule] = nil + var walk func(module.Version) error walk = func(m module.Version) error { _, ok := reqCache[m] @@ -273,7 +277,7 @@ func Req(target module.Version, base []string, reqs Reqs) ([]module.Version, err // UpgradeAll returns a build list for the target module // in which every module is upgraded to its latest version. func UpgradeAll(target module.Version, reqs UpgradeReqs) ([]module.Version, error) { - return buildList(target, reqs, func(m module.Version) (module.Version, error) { + return buildList([]module.Version{target}, reqs, func(m module.Version) (module.Version, error) { if m.Path == target.Path { return target, nil } @@ -308,7 +312,7 @@ func Upgrade(target module.Version, reqs UpgradeReqs, upgrade ...module.Version) } } - return buildList(target, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { + return buildList([]module.Version{target}, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { if v, ok := upgradeTo[m.Path]; ok { return module.Version{Path: m.Path, Version: v}, nil } @@ -331,7 +335,7 @@ func Downgrade(target module.Version, reqs DowngradeReqs, downgrade ...module.Ve // // In order to generate those new requirements, we need to identify versions // for every module in the build list — not just reqs.Required(target). - list, err := BuildList(target, reqs) + list, err := BuildList([]module.Version{target}, reqs) if err != nil { return nil, err } @@ -446,7 +450,7 @@ List: // list with the actual versions of the downgraded modules as selected by MVS, // instead of our initial downgrades. // (See the downhiddenartifact and downhiddencross test cases). - actual, err := BuildList(target, &override{ + actual, err := BuildList([]module.Version{target}, &override{ target: target, list: downgraded, Reqs: reqs, @@ -466,7 +470,7 @@ List: } } - return BuildList(target, &override{ + return BuildList([]module.Version{target}, &override{ target: target, list: downgraded, Reqs: reqs, diff --git a/src/cmd/go/internal/mvs/mvs_test.go b/src/cmd/go/internal/mvs/mvs_test.go index 598ed666889..26d004fee28 100644 --- a/src/cmd/go/internal/mvs/mvs_test.go +++ b/src/cmd/go/internal/mvs/mvs_test.go @@ -507,7 +507,7 @@ func Test(t *testing.T) { t.Fatalf("build takes one argument: %q", line) } fns = append(fns, func(t *testing.T) { - list, err := BuildList(m(kf[1]), reqs) + list, err := BuildList([]module.Version{m(kf[1])}, reqs) checkList(t, key, list, err, val) }) continue diff --git a/src/cmd/go/internal/search/search.go b/src/cmd/go/internal/search/search.go index a0c806a2593..ebd4990a684 100644 --- a/src/cmd/go/internal/search/search.go +++ b/src/cmd/go/internal/search/search.go @@ -202,12 +202,6 @@ func (m *Match) MatchPackages() { } } -var modRoot string - -func SetModRoot(dir string) { - modRoot = dir -} - // MatchDirs sets m.Dirs to a non-nil slice containing all directories that // potentially match a local pattern. The pattern must begin with an absolute // path, or "./", or "../". On Windows, the pattern may use slash or backslash @@ -215,7 +209,7 @@ func SetModRoot(dir string) { // // If any errors may have caused the set of directories to be incomplete, // MatchDirs appends those errors to m.Errs. -func (m *Match) MatchDirs() { +func (m *Match) MatchDirs(modRoots []string) { m.Dirs = []string{} if !m.IsLocal() { m.AddError(fmt.Errorf("internal error: MatchDirs: %s is not a valid filesystem pattern", m.pattern)) @@ -253,15 +247,24 @@ func (m *Match) MatchDirs() { // We need to preserve the ./ for pattern matching // and in the returned import paths. - if modRoot != "" { + if len(modRoots) > 1 { abs, err := filepath.Abs(dir) if err != nil { m.AddError(err) return } - if !hasFilepathPrefix(abs, modRoot) { - m.AddError(fmt.Errorf("directory %s is outside module root (%s)", abs, modRoot)) - return + var found bool + for _, modRoot := range modRoots { + if modRoot != "" && hasFilepathPrefix(abs, modRoot) { + found = true + } + } + if !found { + plural := "" + if len(modRoots) > 1 { + plural = "s" + } + m.AddError(fmt.Errorf("directory %s is outside module root%s (%s)", abs, plural, strings.Join(modRoots, ", "))) } } @@ -424,19 +427,19 @@ func WarnUnmatched(matches []*Match) { // ImportPaths returns the matching paths to use for the given command line. // It calls ImportPathsQuiet and then WarnUnmatched. -func ImportPaths(patterns []string) []*Match { - matches := ImportPathsQuiet(patterns) +func ImportPaths(patterns, modRoots []string) []*Match { + matches := ImportPathsQuiet(patterns, modRoots) WarnUnmatched(matches) return matches } // ImportPathsQuiet is like ImportPaths but does not warn about patterns with no matches. -func ImportPathsQuiet(patterns []string) []*Match { +func ImportPathsQuiet(patterns, modRoots []string) []*Match { var out []*Match for _, a := range CleanPatterns(patterns) { m := NewMatch(a) if m.IsLocal() { - m.MatchDirs() + m.MatchDirs(modRoots) // Change the file import path to a regular import path if the package // is in GOPATH or GOROOT. We don't report errors here; LoadImport From 3cd15e02ed26d86556cb59ff509a1f5a08bca29e Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Fri, 30 Apr 2021 18:09:01 -0400 Subject: [PATCH 04/25] [dev.cmdgo] cmd: pull in x/mod on the dev.cmdgo branch This change changes the requirement on golang.org/x/mod to require the top of the dev.cmdgo branch of that repository. This allows the code on this branch to use the changes on the branch on that repo for parsing go.work files. It also vendors in those changes using go mod vendor. Generated using: go get -d golang.org/x/mod@dev.cmdgo go mod tidy go mod vendor For #45713 Change-Id: I7a20835937c13f90d17c0f39d96b435daec89fba Reviewed-on: https://go-review.googlesource.com/c/go/+/334933 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go.mod | 2 +- src/cmd/go.sum | 4 +- .../vendor/golang.org/x/mod/modfile/rule.go | 250 ++++++++++++------ .../vendor/golang.org/x/mod/modfile/work.go | 234 ++++++++++++++++ src/cmd/vendor/modules.txt | 2 +- 5 files changed, 407 insertions(+), 85 deletions(-) create mode 100644 src/cmd/vendor/golang.org/x/mod/modfile/work.go diff --git a/src/cmd/go.mod b/src/cmd/go.mod index cd03968eedc..21d7d8b75a2 100644 --- a/src/cmd/go.mod +++ b/src/cmd/go.mod @@ -7,7 +7,7 @@ require ( github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 // indirect golang.org/x/arch v0.0.0-20210502124803-cbf565b21d1e golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e // indirect - golang.org/x/mod v0.4.3-0.20210608190319-0f08993efd8a + golang.org/x/mod v0.4.3-0.20210723200715-e41a6a4f3b61 golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744 // indirect golang.org/x/term v0.0.0-20210503060354-a79de5458b56 golang.org/x/tools v0.1.2-0.20210519160823-49064d2332f9 diff --git a/src/cmd/go.sum b/src/cmd/go.sum index d728acaec99..529b152b771 100644 --- a/src/cmd/go.sum +++ b/src/cmd/go.sum @@ -13,8 +13,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e h1:8foAy0aoO5GkqCvAEJ4VC4P3zksTg4X4aJCDpZzmgQI= golang.org/x/crypto v0.0.0-20210503195802-e9a32991a82e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.3-0.20210608190319-0f08993efd8a h1:e8qnjKz4EE6OjRki9wTadWSIogINvq10sMcuBRORxMY= -golang.org/x/mod v0.4.3-0.20210608190319-0f08993efd8a/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.4.3-0.20210723200715-e41a6a4f3b61 h1:gQY3CVezomIImcWCpxp6Mhj+fXCOZ+gD8/88326LVqw= +golang.org/x/mod v0.4.3-0.20210723200715-e41a6a4f3b61/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= diff --git a/src/cmd/vendor/golang.org/x/mod/modfile/rule.go b/src/cmd/vendor/golang.org/x/mod/modfile/rule.go index 78f83fa7144..d6a2d3879ec 100644 --- a/src/cmd/vendor/golang.org/x/mod/modfile/rule.go +++ b/src/cmd/vendor/golang.org/x/mod/modfile/rule.go @@ -423,68 +423,12 @@ func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, a } case "replace": - arrow := 2 - if len(args) >= 2 && args[1] == "=>" { - arrow = 1 - } - if len(args) < arrow+2 || len(args) > arrow+3 || args[arrow] != "=>" { - errorf("usage: %s module/path [v1.2.3] => other/module v1.4\n\t or %s module/path [v1.2.3] => ../local/directory", verb, verb) + replace, wrappederr := parseReplace(f.Syntax.Name, line, verb, args, fix) + if wrappederr != nil { + *errs = append(*errs, *wrappederr) return } - s, err := parseString(&args[0]) - if err != nil { - errorf("invalid quoted string: %v", err) - return - } - pathMajor, err := modulePathMajor(s) - if err != nil { - wrapModPathError(s, err) - return - } - var v string - if arrow == 2 { - v, err = parseVersion(verb, s, &args[1], fix) - if err != nil { - wrapError(err) - return - } - if err := module.CheckPathMajor(v, pathMajor); err != nil { - wrapModPathError(s, err) - return - } - } - ns, err := parseString(&args[arrow+1]) - if err != nil { - errorf("invalid quoted string: %v", err) - return - } - nv := "" - if len(args) == arrow+2 { - if !IsDirectoryPath(ns) { - errorf("replacement module without version must be directory path (rooted or starting with ./ or ../)") - return - } - if filepath.Separator == '/' && strings.Contains(ns, `\`) { - errorf("replacement directory appears to be Windows path (on a non-windows system)") - return - } - } - if len(args) == arrow+3 { - nv, err = parseVersion(verb, ns, &args[arrow+2], fix) - if err != nil { - wrapError(err) - return - } - if IsDirectoryPath(ns) { - errorf("replacement module directory path %q cannot have version", ns) - return - } - } - f.Replace = append(f.Replace, &Replace{ - Old: module.Version{Path: s, Version: v}, - New: module.Version{Path: ns, Version: nv}, - Syntax: line, - }) + f.Replace = append(f.Replace, replace) case "retract": rationale := parseDirectiveComment(block, line) @@ -515,6 +459,83 @@ func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, a } } +func parseReplace(filename string, line *Line, verb string, args []string, fix VersionFixer) (*Replace, *Error) { + wrapModPathError := func(modPath string, err error) *Error { + return &Error{ + Filename: filename, + Pos: line.Start, + ModPath: modPath, + Verb: verb, + Err: err, + } + } + wrapError := func(err error) *Error { + return &Error{ + Filename: filename, + Pos: line.Start, + Err: err, + } + } + errorf := func(format string, args ...interface{}) *Error { + return wrapError(fmt.Errorf(format, args...)) + } + + arrow := 2 + if len(args) >= 2 && args[1] == "=>" { + arrow = 1 + } + if len(args) < arrow+2 || len(args) > arrow+3 || args[arrow] != "=>" { + return nil, errorf("usage: %s module/path [v1.2.3] => other/module v1.4\n\t or %s module/path [v1.2.3] => ../local/directory", verb, verb) + } + s, err := parseString(&args[0]) + if err != nil { + return nil, errorf("invalid quoted string: %v", err) + } + pathMajor, err := modulePathMajor(s) + if err != nil { + return nil, wrapModPathError(s, err) + + } + var v string + if arrow == 2 { + v, err = parseVersion(verb, s, &args[1], fix) + if err != nil { + return nil, wrapError(err) + } + if err := module.CheckPathMajor(v, pathMajor); err != nil { + return nil, wrapModPathError(s, err) + } + } + ns, err := parseString(&args[arrow+1]) + if err != nil { + return nil, errorf("invalid quoted string: %v", err) + } + nv := "" + if len(args) == arrow+2 { + if !IsDirectoryPath(ns) { + return nil, errorf("replacement module without version must be directory path (rooted or starting with ./ or ../)") + } + if filepath.Separator == '/' && strings.Contains(ns, `\`) { + return nil, errorf("replacement directory appears to be Windows path (on a non-windows system)") + } + } + if len(args) == arrow+3 { + nv, err = parseVersion(verb, ns, &args[arrow+2], fix) + if err != nil { + return nil, wrapError(err) + } + if IsDirectoryPath(ns) { + return nil, errorf("replacement module directory path %q cannot have version", ns) + + } + } + return &Replace{ + Old: module.Version{Path: s, Version: v}, + New: module.Version{Path: ns, Version: nv}, + Syntax: line, + }, nil +} + // fixRetract applies fix to each retract directive in f, appending any errors // to errs. // @@ -556,6 +577,63 @@ func (f *File) fixRetract(fix VersionFixer, errs *ErrorList) { } } +func (f *WorkFile) add(errs *ErrorList, line *Line, verb string, args []string, fix VersionFixer) { + wrapError := func(err error) { + *errs = append(*errs, Error{ + Filename: f.Syntax.Name, + Pos: line.Start, + Err: err, + }) + } + errorf := func(format string, args ...interface{}) { + wrapError(fmt.Errorf(format, args...)) + } + + switch verb { + default: + errorf("unknown directive: %s", verb) + + case "go": + if f.Go != nil { + errorf("repeated go statement") + return + } + if len(args) != 1 { + errorf("go directive expects exactly one argument") + return + } else if !GoVersionRE.MatchString(args[0]) { + errorf("invalid go version '%s': must match format 1.23", args[0]) + return + } + + f.Go = &Go{Syntax: line} + f.Go.Version = args[0] + + case "directory": + if len(args) != 1 { + errorf("usage: %s local/dir", verb) + return + } + s, err := parseString(&args[0]) + if err != nil { + errorf("invalid quoted string: %v", err) + return + } + f.Directory = append(f.Directory, &Directory{ + Path: s, + Syntax: line, + }) + + case "replace": + replace, wrappederr := parseReplace(f.Syntax.Name, line, verb, args, fix) + if wrappederr != nil { + *errs = append(*errs, *wrappederr) + return + } + f.Replace = append(f.Replace, replace) + } +} + // IsDirectoryPath reports whether the given path should be interpreted // as a directory path. Just like on the go command line, relative paths // and rooted paths are directory paths; the rest are module paths. @@ -1165,6 +1243,10 @@ func (f *File) DropExclude(path, vers string) error { } func (f *File) AddReplace(oldPath, oldVers, newPath, newVers string) error { + return addReplace(f.Syntax, &f.Replace, oldPath, oldVers, newPath, newVers) +} + +func addReplace(syntax *FileSyntax, replace *[]*Replace, oldPath, oldVers, newPath, newVers string) error { need := true old := module.Version{Path: oldPath, Version: oldVers} new := module.Version{Path: newPath, Version: newVers} @@ -1178,12 +1260,12 @@ func (f *File) AddReplace(oldPath, oldVers, newPath, newVers string) error { } var hint *Line - for _, r := range f.Replace { + for _, r := range *replace { if r.Old.Path == oldPath && (oldVers == "" || r.Old.Version == oldVers) { if need { // Found replacement for old; update to use new. r.New = new - f.Syntax.updateLine(r.Syntax, tokens...) + syntax.updateLine(r.Syntax, tokens...) need = false continue } @@ -1196,7 +1278,7 @@ func (f *File) AddReplace(oldPath, oldVers, newPath, newVers string) error { } } if need { - f.Replace = append(f.Replace, &Replace{Old: old, New: new, Syntax: f.Syntax.addLine(hint, tokens...)}) + *replace = append(*replace, &Replace{Old: old, New: new, Syntax: syntax.addLine(hint, tokens...)}) } return nil } @@ -1282,30 +1364,36 @@ func (f *File) SortBlocks() { // retract directives are not de-duplicated since comments are // meaningful, and versions may be retracted multiple times. func (f *File) removeDups() { + removeDups(f.Syntax, &f.Exclude, &f.Replace) +} + +func removeDups(syntax *FileSyntax, exclude *[]*Exclude, replace *[]*Replace) { kill := make(map[*Line]bool) // Remove duplicate excludes. - haveExclude := make(map[module.Version]bool) - for _, x := range f.Exclude { - if haveExclude[x.Mod] { - kill[x.Syntax] = true - continue + if exclude != nil { + haveExclude := make(map[module.Version]bool) + for _, x := range *exclude { + if haveExclude[x.Mod] { + kill[x.Syntax] = true + continue + } + haveExclude[x.Mod] = true } - haveExclude[x.Mod] = true - } - var excl []*Exclude - for _, x := range f.Exclude { - if !kill[x.Syntax] { - excl = append(excl, x) + var excl []*Exclude + for _, x := range *exclude { + if !kill[x.Syntax] { + excl = append(excl, x) + } } + *exclude = excl } - f.Exclude = excl // Remove duplicate replacements. // Later replacements take priority over earlier ones. haveReplace := make(map[module.Version]bool) - for i := len(f.Replace) - 1; i >= 0; i-- { - x := f.Replace[i] + for i := len(*replace) - 1; i >= 0; i-- { + x := (*replace)[i] if haveReplace[x.Old] { kill[x.Syntax] = true continue @@ -1313,18 +1401,18 @@ func (f *File) removeDups() { haveReplace[x.Old] = true } var repl []*Replace - for _, x := range f.Replace { + for _, x := range *replace { if !kill[x.Syntax] { repl = append(repl, x) } } - f.Replace = repl + *replace = repl // Duplicate require and retract directives are not removed. // Drop killed statements from the syntax tree. var stmts []Expr - for _, stmt := range f.Syntax.Stmt { + for _, stmt := range syntax.Stmt { switch stmt := stmt.(type) { case *Line: if kill[stmt] { @@ -1344,7 +1432,7 @@ func (f *File) removeDups() { } stmts = append(stmts, stmt) } - f.Syntax.Stmt = stmts + syntax.Stmt = stmts } // lineLess returns whether li should be sorted before lj. It sorts diff --git a/src/cmd/vendor/golang.org/x/mod/modfile/work.go b/src/cmd/vendor/golang.org/x/mod/modfile/work.go new file mode 100644 index 00000000000..b1fabff51b7 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/mod/modfile/work.go @@ -0,0 +1,234 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modfile + +import ( + "fmt" + "sort" + "strings" +) + +// A WorkFile is the parsed, interpreted form of a go.work file. +type WorkFile struct { + Go *Go + Directory []*Directory + Replace []*Replace + + Syntax *FileSyntax +} + +// A Directory is a single directory statement. +type Directory struct { + Path string // Directory path of module. + ModulePath string // Module path in the comment. + Syntax *Line +} + +// ParseWork parses and returns a go.work file. +// +// file is the name of the file, used in positions and errors. +// +// data is the content of the file. +// +// fix is an optional function that canonicalizes module versions. +// If fix is nil, all module versions must be canonical (module.CanonicalVersion +// must return the same string). +func ParseWork(file string, data []byte, fix VersionFixer) (*WorkFile, error) { + fs, err := parse(file, data) + if err != nil { + return nil, err + } + f := &WorkFile{ + Syntax: fs, + } + var errs ErrorList + + for _, x := range fs.Stmt { + switch x := x.(type) { + case *Line: + f.add(&errs, x, x.Token[0], x.Token[1:], fix) + + case *LineBlock: + if len(x.Token) > 1 { + errs = append(errs, Error{ + Filename: file, + Pos: x.Start, + Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), + }) + continue + } + switch x.Token[0] { + default: + errs = append(errs, Error{ + Filename: file, + Pos: x.Start, + Err: fmt.Errorf("unknown block type: %s", strings.Join(x.Token, " ")), + }) + continue + case "directory", "replace": + for _, l := range x.Line { + f.add(&errs, l, x.Token[0], l.Token, fix) + } + } + } + } + + if len(errs) > 0 { + return nil, errs + } + return f, nil +} + +// Cleanup cleans up the file f after any edit operations. +// To avoid quadratic behavior, modifications like DropRequire +// clear the entry but do not remove it from the slice. +// Cleanup cleans out all the cleared entries. +func (f *WorkFile) Cleanup() { + w := 0 + for _, r := range f.Directory { + if r.Path != "" { + f.Directory[w] = r + w++ + } + } + f.Directory = f.Directory[:w] + + w = 0 + for _, r := range f.Replace { + if r.Old.Path != "" { + f.Replace[w] = r + w++ + } + } + f.Replace = f.Replace[:w] + + f.Syntax.Cleanup() +} + +func (f *WorkFile) AddGoStmt(version string) error { + if !GoVersionRE.MatchString(version) { + return fmt.Errorf("invalid language version string %q", version) + } + if f.Go == nil { + stmt := &Line{Token: []string{"go", version}} + f.Go = &Go{ + Version: version, + Syntax: stmt, + } + // Find the first non-comment-only block that's and add + // the go statement before it. That will keep file comments at the top. + i := 0 + for i = 0; i < len(f.Syntax.Stmt); i++ { + if _, ok := f.Syntax.Stmt[i].(*CommentBlock); !ok { + break + } + } + f.Syntax.Stmt = append(append(f.Syntax.Stmt[:i:i], stmt), f.Syntax.Stmt[i:]...) + } else { + f.Go.Version = version + f.Syntax.updateLine(f.Go.Syntax, "go", version) + } + return nil +} + +func (f *WorkFile) AddDirectory(diskPath, modulePath string) error { + need := true + for _, d := range f.Directory { + if d.Path == diskPath { + if need { + d.ModulePath = modulePath + f.Syntax.updateLine(d.Syntax, "directory", AutoQuote(diskPath)) + need = false + } else { + d.Syntax.markRemoved() + *d = Directory{} + } + } + } + + if need { + f.AddNewDirectory(diskPath, modulePath) + } + return nil +} + +func (f *WorkFile) AddNewDirectory(diskPath, modulePath string) { + line := f.Syntax.addLine(nil, "directory", AutoQuote(diskPath)) + f.Directory = append(f.Directory, &Directory{Path: diskPath, ModulePath: modulePath, Syntax: line}) +} + +func (f *WorkFile) SetDirectory(dirs []*Directory) { + need := make(map[string]string) + for _, d := range dirs { + need[d.Path] = d.ModulePath + } + + for _, d := range f.Directory { + if modulePath, ok := need[d.Path]; ok { + d.ModulePath = modulePath + } else { + d.Syntax.markRemoved() + *d = Directory{} + } + } + + // TODO(#45713): Add module path to comment. + + for diskPath, modulePath := range need { + f.AddNewDirectory(diskPath, modulePath) + } + f.SortBlocks() +} + +func (f *WorkFile) DropDirectory(path string) error { + for _, d := range f.Directory { + if d.Path == path { + d.Syntax.markRemoved() + *d = Directory{} + } + } + return nil +} + +func (f *WorkFile) AddReplace(oldPath, oldVers, newPath, newVers string) error { + return addReplace(f.Syntax, &f.Replace, oldPath, oldVers, newPath, newVers) +} + +func (f *WorkFile) DropReplace(oldPath, oldVers string) error { + for _, r := range f.Replace { + if r.Old.Path == oldPath && r.Old.Version == oldVers { + r.Syntax.markRemoved() + *r = Replace{} + } + } + return nil +} + +func (f *WorkFile) SortBlocks() { + f.removeDups() // otherwise sorting is unsafe + + for _, stmt := range f.Syntax.Stmt { + block, ok := stmt.(*LineBlock) + if !ok { + continue + } + sort.SliceStable(block.Line, func(i, j int) bool { + return lineLess(block.Line[i], block.Line[j]) + }) + } +} + +// removeDups removes duplicate replace directives. +// +// Later replace directives take priority. +// +// require directives are not de-duplicated. That's left up to higher-level +// logic (MVS). +// +// retract directives are not de-duplicated since comments are +// meaningful, and versions may be retracted multiple times. +func (f *WorkFile) removeDups() { + removeDups(f.Syntax, nil, &f.Replace) +} diff --git a/src/cmd/vendor/modules.txt b/src/cmd/vendor/modules.txt index 34dbdaf5dd3..beceef5392c 100644 --- a/src/cmd/vendor/modules.txt +++ b/src/cmd/vendor/modules.txt @@ -28,7 +28,7 @@ golang.org/x/arch/x86/x86asm ## explicit; go 1.17 golang.org/x/crypto/ed25519 golang.org/x/crypto/ed25519/internal/edwards25519 -# golang.org/x/mod v0.4.3-0.20210608190319-0f08993efd8a +# golang.org/x/mod v0.4.3-0.20210723200715-e41a6a4f3b61 ## explicit; go 1.17 golang.org/x/mod/internal/lazyregexp golang.org/x/mod/modfile From 7ce257147fe0ab3413c8e36909c2408c833efdb8 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 8 Jun 2021 17:07:10 -0400 Subject: [PATCH 05/25] [dev.cmdgo] cmd/go: add the workspace mode This change adds the outline of the implementation of the workspace mode. The go command will now locate go.work files, and read them to determine which modules are in the workspace. It will then put those modules in the root of the workspace when building the build list. It supports building, running, testing, and listing in workspaces. There are still many TODOs for undone work and other changes to fix certain cases. Some of these undone parts include: replaces and go.work.sum files, as well as go mod {test,why,verify}, excludes in workspaces, updating work files to include module names in comments and setting the GOWORK variable. For #45713 Change-Id: I72716af7a300a2896087fc8a79c04e951d248278 Reviewed-on: https://go-review.googlesource.com/c/go/+/334934 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/alldocs.go | 8 + src/cmd/go/internal/base/flag.go | 7 + src/cmd/go/internal/cfg/cfg.go | 6 +- src/cmd/go/internal/envcmd/env.go | 1 + src/cmd/go/internal/list/list.go | 3 + src/cmd/go/internal/modcmd/download.go | 3 + src/cmd/go/internal/modcmd/graph.go | 3 + src/cmd/go/internal/modcmd/verify.go | 3 + src/cmd/go/internal/modcmd/why.go | 2 + src/cmd/go/internal/modload/init.go | 230 +++++++++++++++++++------ src/cmd/go/internal/run/run.go | 3 + src/cmd/go/internal/test/test.go | 2 + src/cmd/go/internal/test/testflag.go | 1 + src/cmd/go/internal/work/build.go | 10 ++ src/cmd/go/testdata/script/work.txt | 71 ++++++++ 15 files changed, 299 insertions(+), 54 deletions(-) create mode 100644 src/cmd/go/testdata/script/work.txt diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 90eb3e2a00b..e7c2e6b51bb 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -167,6 +167,14 @@ // directory, but it is not accessed. When -modfile is specified, an // alternate go.sum file is also used: its path is derived from the // -modfile flag by trimming the ".mod" extension and appending ".sum". +// -workfile file +// in module aware mode, use the given go.work file as a workspace file. +// By default or when -workfile is "auto", the go command searches for a +// file named go.work in the current directory and then containing directories +// until one is found. If a valid go.work file is found, the modules +// specified will collectively be used as the main modules. If -workfile +// is "off", or a go.work file is not found in "auto" mode, workspace +// mode is disabled. // -overlay file // read a JSON config file that provides an overlay for build operations. // The file is a JSON struct with a single field, named 'Replace', that diff --git a/src/cmd/go/internal/base/flag.go b/src/cmd/go/internal/base/flag.go index 677f8196827..2262e2e992b 100644 --- a/src/cmd/go/internal/base/flag.go +++ b/src/cmd/go/internal/base/flag.go @@ -62,6 +62,13 @@ func AddModFlag(flags *flag.FlagSet) { flags.Var(explicitStringFlag{value: &cfg.BuildMod, explicit: &cfg.BuildModExplicit}, "mod", "") } +// AddWorkfileFlag adds the workfile flag to the flag set. It enables workspace +// mode for commands that support it by resetting the cfg.WorkFile variable +// to "" (equivalent to auto) rather than off. +func AddWorkfileFlag(flags *flag.FlagSet) { + flags.Var(explicitStringFlag{value: &cfg.WorkFile, explicit: &cfg.WorkFileExplicit}, "workfile", "") +} + // AddModCommonFlags adds the module-related flags common to build commands // and 'go mod' subcommands. func AddModCommonFlags(flags *flag.FlagSet) { diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 57a3c1ff6fb..da616ee1dd5 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -47,8 +47,10 @@ var ( BuildWork bool // -work flag BuildX bool // -x flag - ModCacheRW bool // -modcacherw flag - ModFile string // -modfile flag + ModCacheRW bool // -modcacherw flag + ModFile string // -modfile flag + WorkFile string // -workfile flag + WorkFileExplicit bool // whether -workfile was set explicitly CmdName string // "build", "install", "list", "mod tidy", etc. diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 1553d263914..f68090f21f3 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -145,6 +145,7 @@ func findEnv(env []cfg.EnvVar, name string) string { // ExtraEnvVars returns environment variables that should not leak into child processes. func ExtraEnvVars() []cfg.EnvVar { gomod := "" + modload.Init() if modload.HasModRoot() { gomod = filepath.Join(modload.ModRoot(), "go.mod") } else if modload.Enabled() { diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 7cb9ec6d949..04630dc341e 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -316,6 +316,7 @@ For more about modules, see https://golang.org/ref/mod. func init() { CmdList.Run = runList // break init cycle work.AddBuildFlags(CmdList, work.DefaultBuildFlags) + base.AddWorkfileFlag(&CmdList.Flag) } var ( @@ -336,6 +337,8 @@ var ( var nl = []byte{'\n'} func runList(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if *listFmt != "" && *listJson == true { base.Fatalf("go list -f cannot be used with -json") } diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 3c88a4b900e..6a99cb01e1e 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -66,6 +66,7 @@ func init() { // TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands. cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "") base.AddModCommonFlags(&cmdDownload.Flag) + base.AddWorkfileFlag(&cmdDownload.Flag) } type moduleJSON struct { @@ -81,6 +82,8 @@ type moduleJSON struct { } func runDownload(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + // Check whether modules are enabled and whether we're in a module. modload.ForceUseModules = true if !modload.HasModRoot() && len(args) == 0 { diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 903bd9970f1..5ef1d4ed04a 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -42,9 +42,12 @@ var ( func init() { cmdGraph.Flag.Var(&graphGo, "go", "") base.AddModCommonFlags(&cmdGraph.Flag) + base.AddWorkfileFlag(&cmdGraph.Flag) } func runGraph(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if len(args) > 0 { base.Fatalf("go mod graph: graph takes no arguments") } diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index 5a6eca32cfb..14c4d76bc36 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -39,9 +39,12 @@ See https://golang.org/ref/mod#go-mod-verify for more about 'go mod verify'. func init() { base.AddModCommonFlags(&cmdVerify.Flag) + base.AddWorkfileFlag(&cmdVerify.Flag) } func runVerify(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if len(args) != 0 { // NOTE(rsc): Could take a module pattern. base.Fatalf("go mod verify: verify takes no arguments") diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index 3b14b27c8c7..eef5fa5ae87 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -61,9 +61,11 @@ var ( func init() { cmdWhy.Run = runWhy // break init cycle base.AddModCommonFlags(&cmdWhy.Flag) + base.AddWorkfileFlag(&cmdWhy.Flag) } func runWhy(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() modload.ForceUseModules = true modload.RootMode = modload.NeedRoot diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 33f91630385..f211e1767c2 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -53,10 +53,6 @@ func TODOWorkspaces(s string) error { var ( initialized bool - // The directory containing go.work file. Set if in a go.work file is found - // and the go command is operating in workspace mode. - workRoot string - // These are primarily used to initialize the MainModules, and should be // eventually superceded by them but are still used in cases where the module // roots are required but MainModules hasn't been initialized yet. Set to @@ -66,6 +62,12 @@ var ( gopath string ) +// Variable set in InitWorkfile +var ( + // Set to the path to the go.work file, or "" if workspace mode is disabled. + workFilePath string +) + type MainModuleSet struct { // versions are the module.Version values of each of the main modules. // For each of them, the Path fields are ordinary module paths and the Version @@ -187,6 +189,20 @@ func BinDir() string { return filepath.Join(gopath, "bin") } +// InitWorkfile initializes the workFilePath variable for commands that +// operate in workspace mode. It should not be called by other commands, +// for example 'go mod tidy', that don't operate in workspace mode. +func InitWorkfile() { + switch cfg.WorkFile { + case "off": + workFilePath = "" + case "", "auto": + workFilePath = findWorkspaceFile(base.Cwd()) + default: + workFilePath = cfg.WorkFile + } +} + // Init determines whether module mode is enabled, locates the root of the // current module (if any), sets environment variables for Git subprocesses, and // configures the cfg, codehost, load, modfetch, and search packages for use @@ -259,6 +275,8 @@ func Init() { base.Fatalf("go: -modfile cannot be used with commands that ignore the current module") } modRoots = nil + } else if inWorkspaceMode() { + // We're in workspace mode. } else { modRoots = findModuleRoots(base.Cwd()) if modRoots == nil { @@ -293,6 +311,7 @@ func Init() { // We're in module mode. Set any global variables that need to be set. cfg.ModulesEnabled = true setDefaultBuildMod() + _ = TODOWorkspaces("ensure that buildmod is readonly") list := filepath.SplitList(cfg.BuildContext.GOPATH) if len(list) == 0 || list[0] == "" { base.Fatalf("missing $GOPATH") @@ -302,7 +321,17 @@ func Init() { base.Fatalf("$GOPATH/go.mod exists but should not") } - if modRoots == nil { + if inWorkspaceMode() { + + _ = TODOWorkspaces("go.work.sum, and also allow modfetch to fall back to individual go.sums") + _ = TODOWorkspaces("replaces") + var err error + modRoots, err = loadWorkFile(workFilePath) + if err != nil { + base.Fatalf("reading go.work: %v", err) + } + // TODO(matloob) should workRoot just be workFile? + } else if modRoots == nil { // We're in module mode, but not inside a module. // // Commands like 'go build', 'go run', 'go list' have no go.mod file to @@ -388,12 +417,24 @@ func ModRoot() string { if !HasModRoot() { die() } + if inWorkspaceMode() { + panic("ModRoot called in workspace mode") + } + // This is similar to MustGetSingleMainModule but we can't call that + // because MainModules may not yet exist when ModRoot is called. if len(modRoots) != 1 { - panic(TODOWorkspaces("need to handle multiple modroots here")) + panic("not in workspace mode but there are multiple ModRoots") } return modRoots[0] } +func inWorkspaceMode() bool { + if !initialized { + panic("inWorkspaceMode called before modload.Init called") + } + return workFilePath != "" +} + // HasModRoot reports whether a main module is present. // HasModRoot may return false even if Enabled returns true: for example, 'get' // does not require a main module. @@ -451,6 +492,31 @@ func (goModDirtyError) Error() string { var errGoModDirty error = goModDirtyError{} +func loadWorkFile(path string) (modRoots []string, err error) { + workDir := filepath.Dir(path) + workData, err := lockedfile.Read(path) + if err != nil { + return nil, err + } + wf, err := modfile.ParseWork(path, workData, nil) + if err != nil { + return nil, err + } + seen := map[string]bool{} + for _, d := range wf.Directory { + modRoot := d.Path + if !filepath.IsAbs(modRoot) { + modRoot = filepath.Join(workDir, modRoot) + } + if seen[modRoot] { + return nil, fmt.Errorf("path %s appears multiple times in workspace", modRoot) + } + seen[modRoot] = true + modRoots = append(modRoots, modRoot) + } + return modRoots, nil +} + // LoadModFile sets Target and, if there is a main module, parses the initial // build list from its go.mod file. // @@ -498,40 +564,62 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { return requirements, false } - gomod := ModFilePath() - data, err := lockedfile.Read(gomod) - if err != nil { - base.Fatalf("go: %v", err) - } - - var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) - if err != nil { - // Errors returned by modfile.Parse begin with file:line. - base.Fatalf("go: errors parsing go.mod:\n%s\n", err) - } - if f.Module == nil { - // No module declaration. Must add module path. - base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod") - } - - // For now, this code assumes there's a single main module, because there's - // no way to specify multiple main modules yet. TODO(#45713): update this - // in a later CL. - modFile = f - mainModule := f.Module.Mod - MainModules = makeMainModules([]module.Version{mainModule}, modRoots) - index = indexModFile(data, f, mainModule, fixed) - - if err := module.CheckImportPath(f.Module.Mod.Path); err != nil { - if pathErr, ok := err.(*module.InvalidPathError); ok { - pathErr.Kind = "module" + var modFiles []*modfile.File + var mainModules []module.Version + for _, modroot := range modRoots { + gomod := modFilePath(modroot) + var data []byte + var err error + if gomodActual, ok := fsys.OverlayPath(gomod); ok { + // Don't lock go.mod if it's part of the overlay. + // On Plan 9, locking requires chmod, and we don't want to modify any file + // in the overlay. See #44700. + data, err = os.ReadFile(gomodActual) + } else { + data, err = lockedfile.Read(gomodActual) + } + if err != nil { + base.Fatalf("go: %v", err) + } + + var fixed bool + f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) + if err != nil { + // Errors returned by modfile.Parse begin with file:line. + base.Fatalf("go: errors parsing go.mod:\n%s\n", err) + } + if f.Module == nil { + // No module declaration. Must add module path. + base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod") + } + + modFile = f // TODO(golang.org/cl/327329): remove the global modFile variable and replace it with multiple modfiles + modFiles = append(modFiles, f) + mainModule := f.Module.Mod + mainModules = append(mainModules, mainModule) + index = indexModFile(data, f, mainModule, fixed) + + if err := module.CheckImportPath(f.Module.Mod.Path); err != nil { + if pathErr, ok := err.(*module.InvalidPathError); ok { + pathErr.Kind = "module" + } + base.Fatalf("go: %v", err) } - base.Fatalf("go: %v", err) } + MainModules = makeMainModules(mainModules, modRoots) setDefaultBuildMod() // possibly enable automatic vendoring - rs = requirementsFromModFile(ctx) + rs = requirementsFromModFiles(ctx, modFiles) + + if inWorkspaceMode() { + // We don't need to do anything for vendor or update the mod file so + // return early. + + _ = TODOWorkspaces("don't worry about commits for now, but eventually will want to update go.work files") + return rs, false + } + + mainModule := MainModules.mustGetSingleMainModule() if cfg.BuildMod == "vendor" { readVendorList() @@ -549,6 +637,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { // Go 1.11 through 1.16 have eager requirements, but the latest Go // version uses lazy requirements instead — so we need to cnvert the // requirements to be lazy. + var err error rs, err = convertDepth(ctx, rs, lazy) if err != nil { base.Fatalf("go: %v", err) @@ -613,7 +702,7 @@ func CreateModFile(ctx context.Context, modPath string) { base.Fatalf("go: %v", err) } - commitRequirements(ctx, modFileGoVersion(), requirementsFromModFile(ctx)) + commitRequirements(ctx, modFileGoVersion(), requirementsFromModFiles(ctx, []*modfile.File{modFile})) // Suggest running 'go mod tidy' unless the project is empty. Even if we // imported all the correct requirements above, we're probably missing @@ -737,29 +826,36 @@ func makeMainModules(ms []module.Version, rootDirs []string) *MainModuleSet { return mainModules } -// requirementsFromModFile returns the set of non-excluded requirements from +// requirementsFromModFiles returns the set of non-excluded requirements from // the global modFile. -func requirementsFromModFile(ctx context.Context) *Requirements { - roots := make([]module.Version, 0, len(modFile.Require)) +func requirementsFromModFiles(ctx context.Context, modFiles []*modfile.File) *Requirements { + rootCap := 0 + for i := range modFiles { + rootCap += len(modFiles[i].Require) + } + roots := make([]module.Version, 0, rootCap) mPathCount := make(map[string]int) for _, m := range MainModules.Versions() { mPathCount[m.Path] = 1 } direct := map[string]bool{} - for _, r := range modFile.Require { - if index != nil && index.exclude[r.Mod] { - if cfg.BuildMod == "mod" { - fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) - } else { - fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + for _, modFile := range modFiles { + // TODO(golang.org/cl/327329): Use the correct index here. + for _, r := range modFile.Require { + if index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + continue } - continue - } - roots = append(roots, r.Mod) - mPathCount[r.Mod.Path]++ - if !r.Indirect { - direct[r.Mod.Path] = true + roots = append(roots, r.Mod) + mPathCount[r.Mod.Path]++ + if !r.Indirect { + direct[r.Mod.Path] = true + } } } module.Sort(roots) @@ -786,6 +882,11 @@ func requirementsFromModFile(ctx context.Context) *Requirements { // wasn't provided. setDefaultBuildMod may be called multiple times. func setDefaultBuildMod() { if cfg.BuildModExplicit { + if inWorkspaceMode() { + base.Fatalf("go: -mod can't be set explicitly when in workspace mode." + + "\n\tRemove the -mod flag to use the default readonly value," + + "\n\tor set -workfile=off to disable workspace mode.") + } // Don't override an explicit '-mod=' argument. return } @@ -944,6 +1045,31 @@ func findModuleRoots(dir string) (roots []string) { return nil } +func findWorkspaceFile(dir string) (root string) { + if dir == "" { + panic("dir not set") + } + dir = filepath.Clean(dir) + + // Look for enclosing go.mod. + for { + f := filepath.Join(dir, "go.work") + if fi, err := fsys.Stat(f); err == nil && !fi.IsDir() { + return f + } + d := filepath.Dir(dir) + if d == dir { + break + } + if d == cfg.GOROOT { + _ = TODOWorkspaces("Address how go.work files interact with GOROOT") + return "" // As a special case, don't cross GOROOT to find a go.work file. + } + dir = d + } + return "" +} + func findAltConfig(dir string) (root, name string) { if dir == "" { panic("dir not set") diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index 784f7162dfd..7d9e2930aba 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -65,6 +65,7 @@ func init() { CmdRun.Run = runRun // break init loop work.AddBuildFlags(CmdRun, work.DefaultBuildFlags) + base.AddWorkfileFlag(&CmdRun.Flag) CmdRun.Flag.Var((*base.StringsFlag)(&work.ExecCmd), "exec", "") } @@ -73,6 +74,8 @@ func printStderr(args ...interface{}) (int, error) { } func runRun(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + if shouldUseOutsideModuleMode(args) { // Set global module flags for 'go run cmd@version'. // This must be done before modload.Init, but we need to call work.BuildInit diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 59ea1ef5445..5fcea18caa9 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -29,6 +29,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/lockedfile" + "cmd/go/internal/modload" "cmd/go/internal/search" "cmd/go/internal/str" "cmd/go/internal/trace" @@ -577,6 +578,7 @@ var defaultVetFlags = []string{ } func runTest(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() pkgArgs, testArgs = testFlags(args) if cfg.DebugTrace != "" { diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index 08f1efa2c0d..f129346d0da 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -29,6 +29,7 @@ import ( func init() { work.AddBuildFlags(CmdTest, work.OmitVFlag) + base.AddWorkfileFlag(&CmdTest.Flag) cf := CmdTest.Flag cf.BoolVar(&testC, "c", false, "") diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 0ed2389cd5a..c51dd398c24 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -121,6 +121,14 @@ and test commands: directory, but it is not accessed. When -modfile is specified, an alternate go.sum file is also used: its path is derived from the -modfile flag by trimming the ".mod" extension and appending ".sum". + -workfile file + in module aware mode, use the given go.work file as a workspace file. + By default or when -workfile is "auto", the go command searches for a + file named go.work in the current directory and then containing directories + until one is found. If a valid go.work file is found, the modules + specified will collectively be used as the main modules. If -workfile + is "off", or a go.work file is not found in "auto" mode, workspace + mode is disabled. -overlay file read a JSON config file that provides an overlay for build operations. The file is a JSON struct with a single field, named 'Replace', that @@ -201,6 +209,7 @@ func init() { AddBuildFlags(CmdBuild, DefaultBuildFlags) AddBuildFlags(CmdInstall, DefaultBuildFlags) + base.AddWorkfileFlag(&CmdBuild.Flag) } // Note that flags consulted by other parts of the code @@ -364,6 +373,7 @@ var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs } var runtimeVersion = runtime.Version() func runBuild(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() BuildInit() var b Builder b.Init() diff --git a/src/cmd/go/testdata/script/work.txt b/src/cmd/go/testdata/script/work.txt new file mode 100644 index 00000000000..f2b51ca6297 --- /dev/null +++ b/src/cmd/go/testdata/script/work.txt @@ -0,0 +1,71 @@ +go run example.com/b +stdout 'Hello from module A' + +# And try from a different directory +cd c +go run example.com/b +stdout 'Hello from module A' +cd $GOPATH/src + +go list all # all includes both modules +stdout 'example.com/a' +stdout 'example.com/b' + +# -mod can't be set in workspace mode, even to readonly +! go list -mod=readonly all +stderr '^go: -mod can''t be set explicitly' +go list -mod=readonly -workfile=off all + +# Test that duplicates in the directory list return an error +cp go.work go.work.backup +cp go.work.dup go.work +! go run example.com/b +stderr 'reading go.work: path .* appears multiple times in workspace' +cp go.work.backup go.work + +-- go.work.dup -- +go 1.17 + +directory ( + a + b + ../src/a +) +-- go.work -- +go 1.17 + +directory ( + ./a + ./b +) + +-- a/go.mod -- + +module example.com/a + +-- a/a.go -- +package a + +import "fmt" + +func HelloFromA() { + fmt.Println("Hello from module A") +} + +-- b/go.mod -- + +module example.com/b + +-- b/main.go -- +package main + +import "example.com/a" + +func main() { + a.HelloFromA() +} + +-- c/README -- +Create this directory so we can cd to +it and make sure paths are interpreted +relative to the go.work, not the cwd. From f05f5ceffa6edec89436a825176eefdd1fe828e5 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 8 Jun 2021 17:07:10 -0400 Subject: [PATCH 06/25] [dev.cmdgo] cmd/go: fold index and modFile into MainModules For #45713 Change-Id: I5e4b0ae16dcc9ba5ac30683370a3a1d3416e24f2 Reviewed-on: https://go-review.googlesource.com/c/go/+/334935 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modload/import.go | 114 +++++++++++++------------ src/cmd/go/internal/modload/init.go | 95 ++++++++++++++++----- src/cmd/go/internal/modload/modfile.go | 85 +++++++++++------- src/cmd/go/internal/modload/query.go | 84 ++++++++++++------ src/cmd/go/internal/modload/vendor.go | 3 +- 5 files changed, 243 insertions(+), 138 deletions(-) diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 7b5305e4bb2..b6b9bf65b8d 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -414,69 +414,71 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M func queryImport(ctx context.Context, path string, rs *Requirements) (module.Version, error) { // To avoid spurious remote fetches, try the latest replacement for each // module (golang.org/issue/26241). - if index != nil { - var mods []module.Version - for mp, mv := range index.highestReplaced { - if !maybeInModule(path, mp) { - continue - } - if mv == "" { - // The only replacement is a wildcard that doesn't specify a version, so - // synthesize a pseudo-version with an appropriate major version and a - // timestamp below any real timestamp. That way, if the main module is - // used from within some other module, the user will be able to upgrade - // the requirement to any real version they choose. - if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { - mv = module.ZeroPseudoVersion(pathMajor[1:]) - } else { - mv = module.ZeroPseudoVersion("v0") + var mods []module.Version + for _, v := range MainModules.Versions() { + if index := MainModules.Index(v); index != nil { + for mp, mv := range index.highestReplaced { + if !maybeInModule(path, mp) { + continue } + if mv == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 { + mv = module.ZeroPseudoVersion(pathMajor[1:]) + } else { + mv = module.ZeroPseudoVersion("v0") + } + } + mg, err := rs.Graph(ctx) + if err != nil { + return module.Version{}, err + } + if cmpVersion(mg.Selected(mp), mv) >= 0 { + // We can't resolve the import by adding mp@mv to the module graph, + // because the selected version of mp is already at least mv. + continue + } + mods = append(mods, module.Version{Path: mp, Version: mv}) } - mg, err := rs.Graph(ctx) - if err != nil { - return module.Version{}, err - } - if cmpVersion(mg.Selected(mp), mv) >= 0 { - // We can't resolve the import by adding mp@mv to the module graph, - // because the selected version of mp is already at least mv. - continue - } - mods = append(mods, module.Version{Path: mp, Version: mv}) } + } - // Every module path in mods is a prefix of the import path. - // As in QueryPattern, prefer the longest prefix that satisfies the import. - sort.Slice(mods, func(i, j int) bool { - return len(mods[i].Path) > len(mods[j].Path) - }) - for _, m := range mods { - needSum := true - root, isLocal, err := fetch(ctx, m, needSum) - if err != nil { - if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { - return module.Version{}, &ImportMissingSumError{importPath: path} - } - return module.Version{}, err - } - if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { - return m, err - } else if ok { - if cfg.BuildMod == "readonly" { - return module.Version{}, &ImportMissingError{Path: path, replaced: m} - } - return m, nil + // Every module path in mods is a prefix of the import path. + // As in QueryPattern, prefer the longest prefix that satisfies the import. + sort.Slice(mods, func(i, j int) bool { + return len(mods[i].Path) > len(mods[j].Path) + }) + for _, m := range mods { + needSum := true + root, isLocal, err := fetch(ctx, m, needSum) + if err != nil { + if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) { + return module.Version{}, &ImportMissingSumError{importPath: path} } + return module.Version{}, err } - if len(mods) > 0 && module.CheckPath(path) != nil { - // The package path is not valid to fetch remotely, - // so it can only exist in a replaced module, - // and we know from the above loop that it is not. - return module.Version{}, &PackageNotInModuleError{ - Mod: mods[0], - Query: "latest", - Pattern: path, - Replacement: Replacement(mods[0]), + if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil { + return m, err + } else if ok { + if cfg.BuildMod == "readonly" { + return module.Version{}, &ImportMissingError{Path: path, replaced: m} } + return m, nil + } + } + if len(mods) > 0 && module.CheckPath(path) != nil { + // The package path is not valid to fetch remotely, + // so it can only exist in a replaced module, + // and we know from the above loop that it is not. + return module.Version{}, &PackageNotInModuleError{ + Mod: mods[0], + Query: "latest", + Pattern: path, + Replacement: Replacement(mods[0]), } } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index f211e1767c2..607054d1ebe 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -17,6 +17,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/cfg" @@ -85,6 +86,11 @@ type MainModuleSet struct { // inGorootSrc caches whether modRoot is within GOROOT/src. // The "std" module is special within GOROOT/src, but not otherwise. inGorootSrc map[module.Version]bool + + modFiles map[module.Version]*modfile.File + + indexMu sync.Mutex + indices map[module.Version]*modFileIndex } func (mms *MainModuleSet) PathPrefix(m module.Version) string { @@ -141,6 +147,36 @@ func (mms *MainModuleSet) mustGetSingleMainModule() module.Version { return mms.versions[0] } +func (mms *MainModuleSet) GetSingleIndexOrNil() *modFileIndex { + if mms == nil { + return nil + } + if len(mms.versions) == 0 { + return nil + } + if len(mms.versions) != 1 { + _ = TODOWorkspaces("Check if we're in workspace mode before returning the below error.") + panic("internal error: mustGetSingleMainModule called in workspace mode") + } + return mms.indices[mms.versions[0]] +} + +func (mms *MainModuleSet) Index(m module.Version) *modFileIndex { + mms.indexMu.Lock() + defer mms.indexMu.Unlock() + return mms.indices[m] +} + +func (mms *MainModuleSet) SetIndex(m module.Version, index *modFileIndex) { + mms.indexMu.Lock() + defer mms.indexMu.Unlock() + mms.indices[m] = index +} + +func (mms *MainModuleSet) ModFile(m module.Version) *modfile.File { + return mms.modFiles[m] +} + func (mms *MainModuleSet) Len() int { if mms == nil { return 0 @@ -178,6 +214,7 @@ const ( // in go.mod, edit it before loading. func ModFile() *modfile.File { Init() + modFile := MainModules.ModFile(MainModules.mustGetSingleMainModule()) if modFile == nil { die() } @@ -557,7 +594,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { if len(modRoots) == 0 { _ = TODOWorkspaces("Instead of creating a fake module with an empty modroot, make MainModules.Len() == 0 mean that we're in module mode but not inside any module.") mainModule := module.Version{Path: "command-line-arguments"} - MainModules = makeMainModules([]module.Version{mainModule}, []string{""}) + MainModules = makeMainModules([]module.Version{mainModule}, []string{""}, []*modfile.File{nil}, []*modFileIndex{nil}) goVersion := LatestGoVersion() rawGoVersion.Store(mainModule, goVersion) requirements = newRequirements(modDepthFromGoVersion(goVersion), nil, nil) @@ -566,6 +603,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { var modFiles []*modfile.File var mainModules []module.Version + var indices []*modFileIndex for _, modroot := range modRoots { gomod := modFilePath(modroot) var data []byte @@ -593,11 +631,10 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { base.Fatalf("go: no module declaration in go.mod. To specify the module path:\n\tgo mod edit -module=example.com/mod") } - modFile = f // TODO(golang.org/cl/327329): remove the global modFile variable and replace it with multiple modfiles modFiles = append(modFiles, f) mainModule := f.Module.Mod mainModules = append(mainModules, mainModule) - index = indexModFile(data, f, mainModule, fixed) + indices = append(indices, indexModFile(data, f, mainModule, fixed)) if err := module.CheckImportPath(f.Module.Mod.Path); err != nil { if pathErr, ok := err.(*module.InvalidPathError); ok { @@ -607,7 +644,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { } } - MainModules = makeMainModules(mainModules, modRoots) + MainModules = makeMainModules(mainModules, modRoots, modFiles, indices) setDefaultBuildMod() // possibly enable automatic vendoring rs = requirementsFromModFiles(ctx, modFiles) @@ -623,14 +660,16 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { if cfg.BuildMod == "vendor" { readVendorList() - checkVendorConsistency() + index := MainModules.Index(mainModule) + modFile := MainModules.ModFile(mainModule) + checkVendorConsistency(index, modFile) rs.initVendor(vendorList) } - if index.goVersionV == "" { + if MainModules.Index(mainModule).goVersionV == "" { // TODO(#45551): Do something more principled instead of checking // cfg.CmdName directly here. if cfg.BuildMod == "mod" && cfg.CmdName != "mod graph" && cfg.CmdName != "mod why" { - addGoStmt(mainModule, LatestGoVersion()) + addGoStmt(MainModules.ModFile(mainModule), mainModule, LatestGoVersion()) if go117EnableLazyLoading { // We need to add a 'go' version to the go.mod file, but we must assume // that its existing contents match something between Go 1.11 and 1.16. @@ -689,12 +728,12 @@ func CreateModFile(ctx context.Context, modPath string) { } fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) - modFile = new(modfile.File) + modFile := new(modfile.File) modFile.AddModuleStmt(modPath) - MainModules = makeMainModules([]module.Version{modFile.Module.Mod}, []string{modRoot}) - addGoStmt(modFile.Module.Mod, LatestGoVersion()) // Add the go directive before converted module requirements. + MainModules = makeMainModules([]module.Version{modFile.Module.Mod}, []string{modRoot}, []*modfile.File{modFile}, []*modFileIndex{nil}) + addGoStmt(modFile, modFile.Module.Mod, LatestGoVersion()) // Add the go directive before converted module requirements. - convertedFrom, err := convertLegacyConfig(modPath) + convertedFrom, err := convertLegacyConfig(modFile, modPath) if convertedFrom != "" { fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(convertedFrom)) } @@ -792,7 +831,7 @@ func AllowMissingModuleImports() { // makeMainModules creates a MainModuleSet and associated variables according to // the given main modules. -func makeMainModules(ms []module.Version, rootDirs []string) *MainModuleSet { +func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile.File, indices []*modFileIndex) *MainModuleSet { for _, m := range ms { if m.Version != "" { panic("mainModulesCalled with module.Version with non empty Version field: " + fmt.Sprintf("%#v", m)) @@ -803,10 +842,14 @@ func makeMainModules(ms []module.Version, rootDirs []string) *MainModuleSet { inGorootSrc: map[module.Version]bool{}, pathPrefix: map[module.Version]string{}, modRoot: map[module.Version]string{}, + modFiles: map[module.Version]*modfile.File{}, + indices: map[module.Version]*modFileIndex{}, } for i, m := range ms { mainModules.pathPrefix[m] = m.Path mainModules.modRoot[m] = rootDirs[i] + mainModules.modFiles[m] = modFiles[i] + mainModules.indices[m] = indices[i] if rel := search.InDir(rootDirs[i], cfg.GOROOTsrc); rel != "" { mainModules.inGorootSrc[m] = true @@ -840,15 +883,18 @@ func requirementsFromModFiles(ctx context.Context, modFiles []*modfile.File) *Re } direct := map[string]bool{} for _, modFile := range modFiles { - // TODO(golang.org/cl/327329): Use the correct index here. + requirement: for _, r := range modFile.Require { - if index != nil && index.exclude[r.Mod] { - if cfg.BuildMod == "mod" { - fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) - } else { - fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + // TODO(#45713): Maybe join + for _, mainModule := range MainModules.Versions() { + if index := MainModules.Index(mainModule); index != nil && index.exclude[r.Mod] { + if cfg.BuildMod == "mod" { + fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } else { + fmt.Fprintf(os.Stderr, "go: ignoring requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version) + } + continue requirement } - continue } roots = append(roots, r.Mod) @@ -908,6 +954,7 @@ func setDefaultBuildMod() { } if len(modRoots) == 1 { + index := MainModules.GetSingleIndexOrNil() if fi, err := fsys.Stat(filepath.Join(modRoots[0], "vendor")); err == nil && fi.IsDir() { modGo := "unspecified" if index != nil && index.goVersionV != "" { @@ -933,7 +980,7 @@ func setDefaultBuildMod() { // convertLegacyConfig imports module requirements from a legacy vendoring // configuration file, if one is present. -func convertLegacyConfig(modPath string) (from string, err error) { +func convertLegacyConfig(modFile *modfile.File, modPath string) (from string, err error) { noneSelected := func(path string) (version string) { return "none" } queryPackage := func(path, rev string) (module.Version, error) { pkgMods, modOnly, err := QueryPattern(context.Background(), path, rev, noneSelected, nil) @@ -967,7 +1014,7 @@ func convertLegacyConfig(modPath string) (from string, err error) { // addGoStmt adds a go directive to the go.mod file if it does not already // include one. The 'go' version added, if any, is the latest version supported // by this toolchain. -func addGoStmt(mod module.Version, v string) { +func addGoStmt(modFile *modfile.File, mod module.Version, v string) { if modFile.Go != nil && modFile.Go.Version != "" { return } @@ -1231,8 +1278,11 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) if MainModules.Len() != 1 || MainModules.ModRoot(MainModules.Versions()[0]) == "" { _ = TODOWorkspaces("also check that workspace mode is off") // We aren't in a module, so we don't have anywhere to write a go.mod file. + _ = TODOWorkspaces("also check that workspace mode is off") return } + mainModule := MainModules.Versions()[0] + modFile := MainModules.ModFile(mainModule) var list []*modfile.Require for _, m := range rs.rootModules { @@ -1251,6 +1301,7 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) } modFile.Cleanup() + index := MainModules.GetSingleIndexOrNil() dirty := index.modFileIsDirty(modFile) if dirty && cfg.BuildMod != "mod" { // If we're about to fail due to -mod=readonly, @@ -1281,7 +1332,7 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) mainModule := MainModules.Versions()[0] // At this point we have determined to make the go.mod file on disk equal to new. - index = indexModFile(new, modFile, mainModule, false) + MainModules.SetIndex(mainModule, indexModFile(new, modFile, mainModule, false)) // Update go.sum after releasing the side lock and refreshing the index. // 'go mod init' shouldn't write go.sum, since it will be incomplete. diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 7b9f6e863aa..f5332ef52f0 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -54,11 +54,13 @@ const ( go117LazyTODO = false ) -var modFile *modfile.File - // modFileGoVersion returns the (non-empty) Go version at which the requirements // in modFile are intepreted, or the latest Go version if modFile is nil. func modFileGoVersion() string { + _ = TODOWorkspaces("this is obviously wrong.") + // Yes we're picking arbitrarily, we'll have to pass through the version + // we care about + modFile := MainModules.ModFile(MainModules.Versions()[0]) if modFile == nil { return LatestGoVersion() } @@ -90,9 +92,6 @@ type modFileIndex struct { exclude map[module.Version]bool } -// index is the index of the go.mod file as of when it was last read or written. -var index *modFileIndex - type requireMeta struct { indirect bool } @@ -135,8 +134,10 @@ var ErrDisallowed = errors.New("disallowed module version") // CheckExclusions returns an error equivalent to ErrDisallowed if module m is // excluded by the main module's go.mod file. func CheckExclusions(ctx context.Context, m module.Version) error { - if index != nil && index.exclude[m] { - return module.VersionError(m, errExcluded) + for _, mainModule := range MainModules.Versions() { + if index := MainModules.Index(mainModule); index != nil && index.exclude[m] { + return module.VersionError(m, errExcluded) + } } return nil } @@ -304,19 +305,37 @@ func CheckDeprecation(ctx context.Context, m module.Version) (deprecation string return summary.deprecated, nil } +func replacement(mod module.Version, index *modFileIndex) (fromVersion string, to module.Version, ok bool) { + if r, ok := index.replace[mod]; ok { + return mod.Version, r, true + } + if r, ok := index.replace[module.Version{Path: mod.Path}]; ok { + return "", r, true + } + return "", module.Version{}, false +} + // Replacement returns the replacement for mod, if any, from go.mod. // If there is no replacement for mod, Replacement returns // a module.Version with Path == "". func Replacement(mod module.Version) module.Version { - if index != nil { - if r, ok := index.replace[mod]; ok { - return r - } - if r, ok := index.replace[module.Version{Path: mod.Path}]; ok { - return r + _ = TODOWorkspaces("support replaces in the go.work file") + foundFrom, found, foundModRoot := "", module.Version{}, "" + for _, v := range MainModules.Versions() { + if index := MainModules.Index(v); index != nil { + if from, r, ok := replacement(mod, index); ok { + modRoot := MainModules.ModRoot(v) + if foundModRoot != "" && foundFrom != from && found != r { + _ = TODOWorkspaces("once the go.work file supports replaces, recommend them as a way to override conflicts") + base.Errorf("conflicting replacements found for %v in workspace modules defined by %v and %v", + mod, modFilePath(foundModRoot), modFilePath(modRoot)) + return found + } + found, foundModRoot = r, modRoot + } } } - return module.Version{} + return found } // resolveReplacement returns the module actually used to load the source code @@ -551,27 +570,29 @@ func goModSummary(m module.Version) (*modFileSummary, error) { } } - if index != nil && len(index.exclude) > 0 { - // Drop any requirements on excluded versions. - // Don't modify the cached summary though, since we might need the raw - // summary separately. - haveExcludedReqs := false - for _, r := range summary.require { - if index.exclude[r] { - haveExcludedReqs = true - break - } - } - if haveExcludedReqs { - s := new(modFileSummary) - *s = *summary - s.require = make([]module.Version, 0, len(summary.require)) + for _, mainModule := range MainModules.Versions() { + if index := MainModules.Index(mainModule); index != nil && len(index.exclude) > 0 { + // Drop any requirements on excluded versions. + // Don't modify the cached summary though, since we might need the raw + // summary separately. + haveExcludedReqs := false for _, r := range summary.require { - if !index.exclude[r] { - s.require = append(s.require, r) + if index.exclude[r] { + haveExcludedReqs = true + break } } - summary = s + if haveExcludedReqs { + s := new(modFileSummary) + *s = *summary + s.require = make([]module.Version, 0, len(summary.require)) + for _, r := range summary.require { + if !index.exclude[r] { + s.require = append(s.require, r) + } + } + summary = s + } } } return summary, nil diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 83e80d009b8..05ef0a9c48f 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -973,14 +973,18 @@ func lookupRepo(proxy, path string) (repo versionRepo, err error) { repo = emptyRepo{path: path, err: err} } - if index == nil { - return repo, err - } - if _, ok := index.highestReplaced[path]; !ok { - return repo, err + // TODO(#45713): Join all the highestReplaced fields into a single value. + for _, mm := range MainModules.Versions() { + index := MainModules.Index(mm) + if index == nil { + continue + } + if _, ok := index.highestReplaced[path]; ok { + return &replacementRepo{repo: repo}, nil + } } - return &replacementRepo{repo: repo}, nil + return repo, err } // An emptyRepo is a versionRepo that contains no versions. @@ -1019,11 +1023,13 @@ func (rr *replacementRepo) Versions(prefix string) ([]string, error) { } versions := repoVersions - if index != nil && len(index.replace) > 0 { - path := rr.ModulePath() - for m, _ := range index.replace { - if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !module.IsPseudoVersion(m.Version) { - versions = append(versions, m.Version) + for _, mm := range MainModules.Versions() { + if index := MainModules.Index(mm); index != nil && len(index.replace) > 0 { + path := rr.ModulePath() + for m, _ := range index.replace { + if m.Path == path && strings.HasPrefix(m.Version, prefix) && m.Version != "" && !module.IsPseudoVersion(m.Version) { + versions = append(versions, m.Version) + } } } } @@ -1041,7 +1047,16 @@ func (rr *replacementRepo) Versions(prefix string) ([]string, error) { func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { info, err := rr.repo.Stat(rev) - if err == nil || index == nil || len(index.replace) == 0 { + if err == nil { + return info, err + } + var hasReplacements bool + for _, v := range MainModules.Versions() { + if index := MainModules.Index(v); index != nil && len(index.replace) > 0 { + hasReplacements = true + } + } + if !hasReplacements { return info, err } @@ -1068,27 +1083,42 @@ func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { func (rr *replacementRepo) Latest() (*modfetch.RevInfo, error) { info, err := rr.repo.Latest() + path := rr.ModulePath() - if index != nil { - path := rr.ModulePath() - if v, ok := index.highestReplaced[path]; ok { - if v == "" { - // The only replacement is a wildcard that doesn't specify a version, so - // synthesize a pseudo-version with an appropriate major version and a - // timestamp below any real timestamp. That way, if the main module is - // used from within some other module, the user will be able to upgrade - // the requirement to any real version they choose. - if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { - v = module.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") - } else { - v = module.PseudoVersion("v0", "", time.Time{}, "000000000000") + highestReplaced, found := "", false + for _, mm := range MainModules.Versions() { + if index := MainModules.Index(mm); index != nil { + if v, ok := index.highestReplaced[path]; ok { + if !found { + highestReplaced, found = v, true + continue + } + if semver.Compare(v, highestReplaced) > 0 { + highestReplaced = v } } + } + } - if err != nil || semver.Compare(v, info.Version) > 0 { - return rr.replacementStat(v) + if found { + v := highestReplaced + + if v == "" { + // The only replacement is a wildcard that doesn't specify a version, so + // synthesize a pseudo-version with an appropriate major version and a + // timestamp below any real timestamp. That way, if the main module is + // used from within some other module, the user will be able to upgrade + // the requirement to any real version they choose. + if _, pathMajor, ok := module.SplitPathVersion(path); ok && len(pathMajor) > 0 { + v = module.PseudoVersion(pathMajor[1:], "", time.Time{}, "000000000000") + } else { + v = module.PseudoVersion("v0", "", time.Time{}, "000000000000") } } + + if err != nil || semver.Compare(v, info.Version) > 0 { + return rr.replacementStat(v) + } } return info, err diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go index e26da15a8f9..6dc8b6cf820 100644 --- a/src/cmd/go/internal/modload/vendor.go +++ b/src/cmd/go/internal/modload/vendor.go @@ -15,6 +15,7 @@ import ( "cmd/go/internal/base" + "golang.org/x/mod/modfile" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -134,7 +135,7 @@ func readVendorList() { // checkVendorConsistency verifies that the vendor/modules.txt file matches (if // go 1.14) or at least does not contradict (go 1.13 or earlier) the // requirements and replacements listed in the main module's go.mod file. -func checkVendorConsistency() { +func checkVendorConsistency(index *modFileIndex, modFile *modfile.File) { readVendorList() pre114 := false From b2205eab0efef6cba784aca4436cb0ef8ac0a4de Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 14 Jun 2021 19:22:58 -0400 Subject: [PATCH 07/25] [dev.cmdgo] cmd/go: add go mod initwork command This command is used to create a go.work file with a set of modules given in the arguments to the command. For #45713 Change-Id: I09f8cefc5849dd43c234dc4a37091791fcc02ebe Reviewed-on: https://go-review.googlesource.com/c/go/+/334936 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/alldocs.go | 18 +++++++++ src/cmd/go/internal/modcmd/initwork.go | 54 ++++++++++++++++++++++++++ src/cmd/go/internal/modcmd/mod.go | 1 + src/cmd/go/internal/modload/init.go | 18 +++++++++ src/cmd/go/testdata/script/work.txt | 10 +++-- 5 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 src/cmd/go/internal/modcmd/initwork.go diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index e7c2e6b51bb..fb99dccb46e 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -1034,6 +1034,7 @@ // edit edit go.mod from tools or scripts // graph print module requirement graph // init initialize new module in current directory +// initwork initialize workspace file // tidy add missing and remove unused modules // vendor make vendored copy of dependencies // verify verify dependencies have expected content @@ -1229,6 +1230,23 @@ // See https://golang.org/ref/mod#go-mod-init for more about 'go mod init'. // // +// Initialize workspace file +// +// Usage: +// +// go mod initwork [moddirs] +// +// go mod initwork initializes and writes a new go.work file in the current +// directory, in effect creating a new workspace at the current directory. +// +// go mod initwork optionally accepts paths to the workspace modules as arguments. +// If the argument is omitted, an empty workspace with no modules will be created. +// +// See the workspaces design proposal at +// https://go.googlesource.com/proposal/+/master/design/45713-workspace.md for +// more information. +// +// // Add missing and remove unused modules // // Usage: diff --git a/src/cmd/go/internal/modcmd/initwork.go b/src/cmd/go/internal/modcmd/initwork.go new file mode 100644 index 00000000000..30653503bcd --- /dev/null +++ b/src/cmd/go/internal/modcmd/initwork.go @@ -0,0 +1,54 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go mod initwork + +package modcmd + +import ( + "cmd/go/internal/base" + "cmd/go/internal/modload" + "context" + "path/filepath" +) + +var _ = modload.TODOWorkspaces("Add more documentation below.T hough this is" + + "enough for those trying workspaces out, there should be more through" + + "documentation if the proposal is accepted.") + +var cmdInitwork = &base.Command{ + UsageLine: "go mod initwork [moddirs]", + Short: "initialize workspace file", + Long: `go mod initwork initializes and writes a new go.work file in the current +directory, in effect creating a new workspace at the current directory. + +go mod initwork optionally accepts paths to the workspace modules as arguments. +If the argument is omitted, an empty workspace with no modules will be created. + +See the workspaces design proposal at +https://go.googlesource.com/proposal/+/master/design/45713-workspace.md for +more information. +`, + Run: runInitwork, +} + +func init() { + base.AddModCommonFlags(&cmdInitwork.Flag) + base.AddWorkfileFlag(&cmdInitwork.Flag) +} + +func runInitwork(ctx context.Context, cmd *base.Command, args []string) { + modload.InitWorkfile() + + modload.ForceUseModules = true + + // TODO(matloob): support using the -workfile path + // To do that properly, we'll have to make the module directories + // make dirs relative to workFile path before adding the paths to + // the directory entries + + workFile := filepath.Join(base.Cwd(), "go.work") + + modload.CreateWorkFile(ctx, workFile, args) +} diff --git a/src/cmd/go/internal/modcmd/mod.go b/src/cmd/go/internal/modcmd/mod.go index d72d0cacd68..3586b44c1ad 100644 --- a/src/cmd/go/internal/modcmd/mod.go +++ b/src/cmd/go/internal/modcmd/mod.go @@ -25,6 +25,7 @@ See 'go help modules' for an overview of module functionality. cmdEdit, cmdGraph, cmdInit, + cmdInitwork, cmdTidy, cmdVendor, cmdVerify, diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 607054d1ebe..18f0f2b8f86 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -767,6 +767,24 @@ func CreateModFile(ctx context.Context, modPath string) { } } +// CreateWorkFile initializes a new workspace by creating a go.work file. +func CreateWorkFile(ctx context.Context, workFile string, modDirs []string) { + _ = TODOWorkspaces("Report an error if the file already exists.") + + goV := LatestGoVersion() // Use current Go version by default + workF := new(modfile.WorkFile) + workF.Syntax = new(modfile.FileSyntax) + workF.AddGoStmt(goV) + + for _, dir := range modDirs { + _ = TODOWorkspaces("Add the module path of the module.") + workF.AddDirectory(dir, "") + } + + data := modfile.Format(workF.Syntax) + lockedfile.Write(workFile, bytes.NewReader(data), 0644) +} + // fixVersion returns a modfile.VersionFixer implemented using the Query function. // // It resolves commit hashes and branch names to versions, diff --git a/src/cmd/go/testdata/script/work.txt b/src/cmd/go/testdata/script/work.txt index f2b51ca6297..c68ca89a767 100644 --- a/src/cmd/go/testdata/script/work.txt +++ b/src/cmd/go/testdata/script/work.txt @@ -1,3 +1,6 @@ +go mod initwork ./a ./b +cmp go.work go.work.want + go run example.com/b stdout 'Hello from module A' @@ -31,14 +34,13 @@ directory ( b ../src/a ) --- go.work -- +-- go.work.want -- go 1.17 directory ( - ./a - ./b + ./a + ./b ) - -- a/go.mod -- module example.com/a From 72233d27c4dcbbbbb53f06bdafd4a0fb8d652662 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Tue, 20 Jul 2021 14:37:53 -0700 Subject: [PATCH 08/25] [dev.cmdgo] cmd/go: add -testsum flag to update go.sum in script tests -testsum may be set to "tidy", "listm", or "listall". When set, TestScript runs 'go mod tidy', 'go list -m -mod=mod all', or 'go list -mod=mod all' at the beginning of each test that has a go.mod file in its root directory. If the test passes and go.mod or go.sum was updated, TestScript will rewrite the test file with the initial content of go.mod and go.sum (after the above command). This is useful for writing tests that need a working go.sum and for fixing tests that rely on -mod=mod. For golang/go#41302 Change-Id: I63a5667621a5082ccedfc1bff33c3969c29e8b3d Reviewed-on: https://go-review.googlesource.com/c/go/+/336150 Run-TryBot: Jay Conrod TryBot-Result: Go Bot Trust: Jay Conrod Reviewed-by: Michael Matloob --- src/cmd/go/script_test.go | 81 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index 639e907db07..9ca297e89b4 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -11,6 +11,7 @@ import ( "bytes" "context" "errors" + "flag" "fmt" "go/build" "internal/testenv" @@ -35,6 +36,8 @@ import ( "cmd/internal/sys" ) +var testSum = flag.String("testsum", "", `may be tidy, listm, or listall. If set, TestScript generates a go.sum file at the beginning of each test and updates test files if they pass.`) + // TestScript runs the tests in testdata/script/*.txt. func TestScript(t *testing.T) { testenv.MustHaveGoBuild(t) @@ -269,6 +272,22 @@ func (ts *testScript) run() { ts.mark = ts.log.Len() } + // With -testsum, if a go.mod file is present in the test's initial + // working directory, run 'go mod tidy'. + if *testSum != "" { + if ts.updateSum(a) { + defer func() { + if ts.t.Failed() { + return + } + data := txtar.Format(a) + if err := os.WriteFile(ts.file, data, 0666); err != nil { + ts.t.Errorf("rewriting test file: %v", err) + } + }() + } + } + // Run script. // See testdata/script/README for documentation of script form. script := string(a.Comment) @@ -1341,6 +1360,68 @@ func (ts *testScript) parse(line string) command { return cmd } +// updateSum runs 'go mod tidy', 'go list -mod=mod -m all', or +// 'go list -mod=mod all' in the test's current directory if a file named +// "go.mod" is present after the archive has been extracted. updateSum modifies +// archive and returns true if go.mod or go.sum were changed. +func (ts *testScript) updateSum(archive *txtar.Archive) (rewrite bool) { + gomodIdx, gosumIdx := -1, -1 + for i := range archive.Files { + switch archive.Files[i].Name { + case "go.mod": + gomodIdx = i + case "go.sum": + gosumIdx = i + } + } + if gomodIdx < 0 { + return false + } + + switch *testSum { + case "tidy": + ts.cmdGo(success, []string{"mod", "tidy"}) + case "listm": + ts.cmdGo(success, []string{"list", "-m", "-mod=mod", "all"}) + case "listall": + ts.cmdGo(success, []string{"list", "-mod=mod", "all"}) + default: + ts.t.Fatalf(`unknown value for -testsum %q; may be "tidy", "listm", or "listall"`, *testSum) + } + + newGomodData, err := os.ReadFile(filepath.Join(ts.cd, "go.mod")) + if err != nil { + ts.t.Fatalf("reading go.mod after -testsum: %v", err) + } + if !bytes.Equal(newGomodData, archive.Files[gomodIdx].Data) { + archive.Files[gomodIdx].Data = newGomodData + rewrite = true + } + + newGosumData, err := os.ReadFile(filepath.Join(ts.cd, "go.sum")) + if err != nil && !os.IsNotExist(err) { + ts.t.Fatalf("reading go.sum after -testsum: %v", err) + } + switch { + case os.IsNotExist(err) && gosumIdx >= 0: + // go.sum was deleted. + rewrite = true + archive.Files = append(archive.Files[:gosumIdx], archive.Files[gosumIdx+1:]...) + case err == nil && gosumIdx < 0: + // go.sum was created. + rewrite = true + gosumIdx = gomodIdx + 1 + archive.Files = append(archive.Files, txtar.File{}) + copy(archive.Files[gosumIdx+1:], archive.Files[gosumIdx:]) + archive.Files[gosumIdx] = txtar.File{Name: "go.sum", Data: newGosumData} + case err == nil && gosumIdx >= 0 && !bytes.Equal(newGosumData, archive.Files[gosumIdx].Data): + // go.sum was changed. + rewrite = true + archive.Files[gosumIdx].Data = newGosumData + } + return rewrite +} + // diff returns a formatted diff of the two texts, // showing the entire text and the minimum line-level // additions and removals to turn text1 into text2. From 2c8acf63c233430f8fb48f37c6ec54a29bd53c28 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Tue, 20 Jul 2021 14:46:40 -0700 Subject: [PATCH 09/25] [dev.cmdgo] cmd/go: make fewer 'go mod' commands update go.mod 'go mod graph', 'go mod vendor', 'go mod verify', and 'go mod why' will no longer edit go.mod or go.sum. 'go mod graph', 'go mod verify', and 'go mod why' may still fetch files and look up packages as if they were able to update go.mod. They're useful for debugging and should still work when go.mod is a little broken. This is implemented in modload.setDefaultBuildMod based on command name for now. Super gross. Sorry. This should be fixed with a larger refactoring for #40775. Fixes golang/go#45551 Change-Id: If5f225937180d32e9a5dd252c78d988042bbdedf Reviewed-on: https://go-review.googlesource.com/c/go/+/336151 Trust: Jay Conrod Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Michael Matloob --- src/cmd/go/internal/modload/init.go | 21 +++++++++++++---- src/cmd/go/testdata/script/mod_all.txt | 2 +- src/cmd/go/testdata/script/mod_e.txt | 23 ++++++++++++------- src/cmd/go/testdata/script/mod_get_commit.txt | 2 +- .../go/testdata/script/mod_getmode_vendor.txt | 1 + .../go/testdata/script/mod_list_retract.txt | 4 +++- src/cmd/go/testdata/script/mod_retention.txt | 8 ++++--- src/cmd/go/testdata/script/mod_tidy_error.txt | 4 ++-- .../go/testdata/script/mod_vendor_replace.txt | 5 +++- .../testdata/script/mod_vendor_trimpath.txt | 6 ++++- .../script/mod_vendor_unused_only.txt | 2 ++ src/cmd/go/testdata/script/mod_verify.txt | 5 ---- src/cmd/go/testdata/script/modfile_flag.txt | 5 ++++ 13 files changed, 61 insertions(+), 27 deletions(-) diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 18f0f2b8f86..37587865623 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -955,12 +955,25 @@ func setDefaultBuildMod() { return } - if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") { - // 'get' and 'go mod' commands may update go.mod automatically. - // TODO(jayconrod): should this narrower? Should 'go mod download' or - // 'go mod graph' update go.mod by default? + // TODO(#40775): commands should pass in the module mode as an option + // to modload functions instead of relying on an implicit setting + // based on command name. + switch cfg.CmdName { + case "get", "mod download", "mod init", "mod tidy": + // These commands are intended to update go.mod and go.sum. cfg.BuildMod = "mod" return + case "mod graph", "mod verify", "mod why": + // These commands should not update go.mod or go.sum, but they should be + // able to fetch modules not in go.sum and should not report errors if + // go.mod is inconsistent. They're useful for debugging, and they need + // to work in buggy situations. + cfg.BuildMod = "mod" + allowWriteGoMod = false + return + case "mod vendor": + cfg.BuildMod = "readonly" + return } if modRoots == nil { if allowMissingModuleImports { diff --git a/src/cmd/go/testdata/script/mod_all.txt b/src/cmd/go/testdata/script/mod_all.txt index 090eeee22df..6fa2d832396 100644 --- a/src/cmd/go/testdata/script/mod_all.txt +++ b/src/cmd/go/testdata/script/mod_all.txt @@ -315,7 +315,7 @@ go 1.15 require ( example.com/a v0.1.0 - example.com/b v0.1.0 + example.com/b v0.1.0 // indirect example.com/q v0.1.0 example.com/r v0.1.0 // indirect example.com/t v0.1.0 diff --git a/src/cmd/go/testdata/script/mod_e.txt b/src/cmd/go/testdata/script/mod_e.txt index 3a0d18dabc2..3cffaf6ef1c 100644 --- a/src/cmd/go/testdata/script/mod_e.txt +++ b/src/cmd/go/testdata/script/mod_e.txt @@ -24,11 +24,11 @@ cmp go.mod.orig go.mod ! go mod vendor -stderr '^example.com/untidy imports\n\texample.net/directnotfound: cannot find module providing package example.net/directnotfound: module example.net/directnotfound: reading http://.*: 404 Not Found$' +stderr '^example.com/untidy imports\n\texample.net/directnotfound: no required module provides package example.net/directnotfound; to add it:\n\tgo get example.net/directnotfound$' -stderr '^example.com/untidy imports\n\texample.net/m imports\n\texample.net/indirectnotfound: cannot find module providing package example.net/indirectnotfound: module example.net/indirectnotfound: reading http://.*: 404 Not Found$' +stderr '^example.com/untidy imports\n\texample.net/m: module example.net/m provides package example.net/m and is replaced but not required; to add it:\n\tgo get example.net/m@v0.1.0$' -stderr '^example.com/untidy tested by\n\texample.com/untidy.test imports\n\texample.net/directtestnotfound: cannot find module providing package example.net/directtestnotfound: module example.net/directtestnotfound: reading http://.*: 404 Not Found$' +stderr '^example.com/untidy tested by\n\texample.com/untidy.test imports\n\texample.net/directtestnotfound: no required module provides package example.net/directtestnotfound; to add it:\n\tgo get example.net/directtestnotfound$' ! stderr 'indirecttestnotfound' # Vendor prunes test dependencies. @@ -43,16 +43,23 @@ stderr -count=4 'cannot find module providing package' cmp go.mod.final go.mod -# 'go mod vendor -e' still logs the errors, but succeeds and updates go.mod. - +# 'go mod vendor -e' still logs the errors, but creates a vendor directory +# and exits with status 0. +# 'go mod vendor -e' does not update go.mod and will not vendor packages that +# would require changing go.mod, for example, by adding a requirement. cp go.mod.orig go.mod go mod vendor -e -stderr -count=3 'cannot find module providing package' -cmp go.mod.final go.mod +stderr -count=2 'no required module provides package' +stderr '^example.com/untidy imports\n\texample.net/m: module example.net/m provides package example.net/m and is replaced but not required; to add it:\n\tgo get example.net/m@v0.1.0$' +exists vendor/modules.txt +! exists vendor/example.net + +go mod edit -require example.net/m@v0.1.0 +go mod vendor -e +stderr -count=3 'no required module provides package' exists vendor/modules.txt exists vendor/example.net/m/m.go - -- go.mod -- module example.com/untidy go 1.16 diff --git a/src/cmd/go/testdata/script/mod_get_commit.txt b/src/cmd/go/testdata/script/mod_get_commit.txt index 4649491a532..0cf94ae1821 100644 --- a/src/cmd/go/testdata/script/mod_get_commit.txt +++ b/src/cmd/go/testdata/script/mod_get_commit.txt @@ -44,7 +44,7 @@ go mod edit -require rsc.io/quote@23179ee grep 'rsc.io/quote 23179ee' go.mod # but other commands fix them -go mod graph +go list -m -mod=mod all grep 'rsc.io/quote v1.5.1' go.mod -- go.mod -- diff --git a/src/cmd/go/testdata/script/mod_getmode_vendor.txt b/src/cmd/go/testdata/script/mod_getmode_vendor.txt index d3df2078b07..00070c03b53 100644 --- a/src/cmd/go/testdata/script/mod_getmode_vendor.txt +++ b/src/cmd/go/testdata/script/mod_getmode_vendor.txt @@ -25,6 +25,7 @@ stderr 'go list -m: can''t match module patterns using the vendor directory\n\t\ -- go.mod -- module x +go 1.16 -- x.go -- package x import _ "rsc.io/quote" diff --git a/src/cmd/go/testdata/script/mod_list_retract.txt b/src/cmd/go/testdata/script/mod_list_retract.txt index 4b133485152..b7147aa1824 100644 --- a/src/cmd/go/testdata/script/mod_list_retract.txt +++ b/src/cmd/go/testdata/script/mod_list_retract.txt @@ -101,7 +101,9 @@ module example.com/use go 1.15 require example.com/retract v1.0.0-bad - +-- go.sum -- +example.com/retract v1.0.0-bad h1:liAW69rbtjY67x2CcNzat668L/w+YGgNX3lhJsWIJis= +example.com/retract v1.0.0-bad/go.mod h1:0DvGGofJ9hr1q63cBrOY/jSY52OwhRGA0K47NE80I5Y= -- use.go -- package use diff --git a/src/cmd/go/testdata/script/mod_retention.txt b/src/cmd/go/testdata/script/mod_retention.txt index 7a371b18068..481c10d2b7d 100644 --- a/src/cmd/go/testdata/script/mod_retention.txt +++ b/src/cmd/go/testdata/script/mod_retention.txt @@ -39,12 +39,14 @@ go list -mod=mod all cmp go.mod go.mod.tidy # "// indirect" comments should be added if appropriate. +# TODO(#42504): add case for 'go list -mod=mod -tags=any all' when -tags=any +# is supported. Only a command that loads "all" without build constraints +# (except "ignore") has enough information to add "// indirect" comments. +# 'go mod tidy' and 'go mod vendor' are the only commands that do that, +# but 'go mod vendor' cannot write go.mod. cp go.mod.toodirect go.mod go list all cmp go.mod go.mod.toodirect -go mod vendor # loads everything, so adds "// indirect" comments. -cmp go.mod go.mod.tidy -rm -r vendor # Redundant requirements should be preserved... diff --git a/src/cmd/go/testdata/script/mod_tidy_error.txt b/src/cmd/go/testdata/script/mod_tidy_error.txt index 395537b1a71..51fc65fa7a8 100644 --- a/src/cmd/go/testdata/script/mod_tidy_error.txt +++ b/src/cmd/go/testdata/script/mod_tidy_error.txt @@ -10,8 +10,8 @@ stderr '^issue27063 imports\n\tissue27063/other imports\n\tother.example.com/non ! go mod vendor ! stderr 'package nonexist is not in GOROOT' -stderr '^issue27063 imports\n\tnonexist.example.com: cannot find module providing package nonexist.example.com' -stderr '^issue27063 imports\n\tissue27063/other imports\n\tother.example.com/nonexist: cannot find module providing package other.example.com/nonexist' +stderr '^issue27063 imports\n\tnonexist.example.com: no required module provides package nonexist.example.com; to add it:\n\tgo get nonexist.example.com$' +stderr '^issue27063 imports\n\tissue27063/other imports\n\tother.example.com/nonexist: no required module provides package other.example.com/nonexist; to add it:\n\tgo get other.example.com/nonexist$' -- go.mod -- module issue27063 diff --git a/src/cmd/go/testdata/script/mod_vendor_replace.txt b/src/cmd/go/testdata/script/mod_vendor_replace.txt index 0c1c1d22f5b..1820af62ad8 100644 --- a/src/cmd/go/testdata/script/mod_vendor_replace.txt +++ b/src/cmd/go/testdata/script/mod_vendor_replace.txt @@ -36,7 +36,6 @@ module example.com/replace require rsc.io/quote/v3 v3.0.0 replace rsc.io/quote/v3 => ./local/not-rsc.io/quote/v3 - -- imports.go -- package replace @@ -64,3 +63,7 @@ require ( not-rsc.io/quote/v3 v3.0.0 ) replace not-rsc.io/quote/v3 => rsc.io/quote/v3 v3.0.0 +-- multiple-paths/go.sum -- +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +rsc.io/quote/v3 v3.0.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/src/cmd/go/testdata/script/mod_vendor_trimpath.txt b/src/cmd/go/testdata/script/mod_vendor_trimpath.txt index 5451aa773c8..d9d9d988974 100644 --- a/src/cmd/go/testdata/script/mod_vendor_trimpath.txt +++ b/src/cmd/go/testdata/script/mod_vendor_trimpath.txt @@ -29,8 +29,12 @@ stdout '^example.com/stack@v1.0.0/stack.go$' -- go.mod -- module example.com/main -require example.com/stack v1.0.0 +go 1.17 +require example.com/stack v1.0.0 +-- go.sum -- +example.com/stack v1.0.0 h1:IEDLeew5NytZ8vrgCF/QVem3H3SR3QMttdu9HfJvk9I= +example.com/stack v1.0.0/go.mod h1:7wFEbaV5e5O7wJ8aBdqQOR//UXppm/pwnwziMKViuI4= -- main.go -- package main diff --git a/src/cmd/go/testdata/script/mod_vendor_unused_only.txt b/src/cmd/go/testdata/script/mod_vendor_unused_only.txt index 839c6453cf8..accd9f373de 100644 --- a/src/cmd/go/testdata/script/mod_vendor_unused_only.txt +++ b/src/cmd/go/testdata/script/mod_vendor_unused_only.txt @@ -12,6 +12,8 @@ module example.com/m go 1.14 require example.com v1.0.0 // indirect +-- go.sum -- +example.com v1.0.0/go.mod h1:WRiieAqDBb1hVdDXLLdxNtCDWNfehn7FWyPC5Oz2vB4= -- go1.14-modules.txt -- # example.com v1.0.0 ## explicit diff --git a/src/cmd/go/testdata/script/mod_verify.txt b/src/cmd/go/testdata/script/mod_verify.txt index b5106659a9a..f02d15aa289 100644 --- a/src/cmd/go/testdata/script/mod_verify.txt +++ b/src/cmd/go/testdata/script/mod_verify.txt @@ -39,11 +39,6 @@ stderr 'go.mod: checksum mismatch' # go.sum should be created and updated automatically. rm go.sum -go mod graph -exists go.sum -grep '^rsc.io/quote v1.1.0/go.mod ' go.sum -! grep '^rsc.io/quote v1.1.0 ' go.sum - go mod tidy grep '^rsc.io/quote v1.1.0/go.mod ' go.sum grep '^rsc.io/quote v1.1.0 ' go.sum diff --git a/src/cmd/go/testdata/script/modfile_flag.txt b/src/cmd/go/testdata/script/modfile_flag.txt index 5852c311891..7cce581e551 100644 --- a/src/cmd/go/testdata/script/modfile_flag.txt +++ b/src/cmd/go/testdata/script/modfile_flag.txt @@ -24,6 +24,11 @@ stdout '^go.alt.mod$' go mod edit -require rsc.io/quote@v1.5.2 grep rsc.io/quote go.alt.mod +# 'go list -m' should add sums to the alternate go.sum. +go list -m -mod=mod all +grep '^rsc.io/quote v1.5.2/go.mod ' go.alt.sum +! grep '^rsc.io/quote v1.5.2 ' go.alt.sum + # other 'go mod' commands should work. 'go mod vendor' is tested later. go mod download rsc.io/quote go mod graph From 288a83dcffef18514e8c01f0ca2053c6be185305 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 28 Jun 2021 15:48:03 -0400 Subject: [PATCH 10/25] [dev.cmdgo] cmd/go: maintain a go.work.sum file This change causes the go command to maintain a separate go.work.sum file when in workspace mode rather than using the go.sum files from the individual modules. This isn't quite what the proposal spec specifies, which is that the sums that don't exist in any of the workspace modules are added to go.work.sum rather than the necessary sums. That will be done in a future change. Change-Id: I528b9b153a93a4cd67c5af471ad6d5bd3628578b Reviewed-on: https://go-review.googlesource.com/c/go/+/334939 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modfetch/fetch.go | 15 ++++---- src/cmd/go/internal/modload/buildlist.go | 4 +-- src/cmd/go/internal/modload/import.go | 4 ++- src/cmd/go/internal/modload/init.go | 24 ++++++++++--- src/cmd/go/internal/modload/load.go | 4 ++- src/cmd/go/internal/modload/modfile.go | 2 +- src/cmd/go/testdata/script/work.txt | 46 +++++++++++++++++++++--- src/cmd/go/testdata/script/work_sum.txt | 33 +++++++++++++++++ 8 files changed, 113 insertions(+), 19 deletions(-) create mode 100644 src/cmd/go/testdata/script/work_sum.txt diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e40593abae8..7b3525e9143 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -681,19 +681,21 @@ func isValidSum(data []byte) bool { return true } +var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly") + // WriteGoSum writes the go.sum file if it needs to be updated. // // keep is used to check whether a newly added sum should be saved in go.sum. // It should have entries for both module content sums and go.mod sums // (version ends with "/go.mod"). Existing sums will be preserved unless they // have been marked for deletion with TrimGoSum. -func WriteGoSum(keep map[module.Version]bool) { +func WriteGoSum(keep map[module.Version]bool, readonly bool) error { goSum.mu.Lock() defer goSum.mu.Unlock() // If we haven't read the go.sum file yet, don't bother writing it. if !goSum.enabled { - return + return nil } // Check whether we need to add sums for which keep[m] is true or remove @@ -711,10 +713,10 @@ Outer: } } if !dirty { - return + return nil } - if cfg.BuildMod == "readonly" { - base.Fatalf("go: updates to go.sum needed, disabled by -mod=readonly") + if readonly { + return ErrGoSumDirty } // Make a best-effort attempt to acquire the side lock, only to exclude @@ -759,11 +761,12 @@ Outer: }) if err != nil { - base.Fatalf("go: updating go.sum: %v", err) + return fmt.Errorf("updating go.sum: %w", err) } goSum.status = make(map[modSum]modSumStatus) goSum.overwrite = false + return nil } // TrimGoSum trims go.sum to contain only the modules needed for reproducible diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 959ee25df4e..d2957a34c42 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -807,7 +807,7 @@ func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requiremen // We've added or upgraded one or more roots, so load the full module // graph so that we can update those roots to be consistent with other // requirements. - if cfg.BuildMod != "mod" { + if mustHaveCompleteRequirements() { // Our changes to the roots may have moved dependencies into or out of // the lazy-loading horizon, which could in turn change the selected // versions of other modules. (Unlike for eager modules, for lazy @@ -1007,7 +1007,7 @@ func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requireme return rs, err } - if cfg.BuildMod != "mod" { + if mustHaveCompleteRequirements() { // Instead of actually updating the requirements, just check that no updates // are needed. if rs == nil { diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index b6b9bf65b8d..773d8b600b2 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -32,6 +32,8 @@ type ImportMissingError struct { Module module.Version QueryErr error + ImportingModule module.Version + // isStd indicates whether we would expect to find the package in the standard // library. This is normally true for all dotless import paths, but replace // directives can cause us to treat the replaced paths as also being in @@ -673,7 +675,7 @@ func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, i mod = r } - if HasModRoot() && cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) { + if HasModRoot() && cfg.BuildMod == "readonly" && !inWorkspaceMode() && needSum && !modfetch.HaveSum(mod) { return "", false, module.VersionError(mod, &sumMissingError{}) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 37587865623..00dfc8b2dc5 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -367,6 +367,7 @@ func Init() { if err != nil { base.Fatalf("reading go.work: %v", err) } + modfetch.GoSumFile = workFilePath + ".sum" // TODO(matloob) should workRoot just be workFile? } else if modRoots == nil { // We're in module mode, but not inside a module. @@ -1009,6 +1010,10 @@ func setDefaultBuildMod() { cfg.BuildMod = "readonly" } +func mustHaveCompleteRequirements() bool { + return cfg.BuildMod != "mod" && !inWorkspaceMode() +} + // convertLegacyConfig imports module requirements from a legacy vendoring // configuration file, if one is present. func convertLegacyConfig(modFile *modfile.File, modPath string) (from string, err error) { @@ -1306,10 +1311,17 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) return } + if inWorkspaceMode() { + // go.mod files aren't updated in workspace mode, but we still want to + // update the go.work.sum file. + if err := modfetch.WriteGoSum(keepSums(ctx, loaded, rs, addBuildListZipSums), mustHaveCompleteRequirements()); err != nil { + base.Fatalf("go: %v", err) + } + return + } + if MainModules.Len() != 1 || MainModules.ModRoot(MainModules.Versions()[0]) == "" { - _ = TODOWorkspaces("also check that workspace mode is off") // We aren't in a module, so we don't have anywhere to write a go.mod file. - _ = TODOWorkspaces("also check that workspace mode is off") return } mainModule := MainModules.Versions()[0] @@ -1346,7 +1358,9 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) // Don't write go.mod, but write go.sum in case we added or trimmed sums. // 'go mod init' shouldn't write go.sum, since it will be incomplete. if cfg.CmdName != "mod init" { - modfetch.WriteGoSum(keepSums(ctx, loaded, rs, addBuildListZipSums)) + if err := modfetch.WriteGoSum(keepSums(ctx, loaded, rs, addBuildListZipSums), mustHaveCompleteRequirements()); err != nil { + base.Fatalf("go: %v", err) + } } return } @@ -1368,7 +1382,9 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) // Update go.sum after releasing the side lock and refreshing the index. // 'go mod init' shouldn't write go.sum, since it will be incomplete. if cfg.CmdName != "mod init" { - modfetch.WriteGoSum(keepSums(ctx, loaded, rs, addBuildListZipSums)) + if err := modfetch.WriteGoSum(keepSums(ctx, loaded, rs, addBuildListZipSums), mustHaveCompleteRequirements()); err != nil { + base.Fatalf("go: %v", err) + } } }() diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 77d2dc40303..e7b03b08e9d 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -403,7 +403,9 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma // loaded.requirements, but here we may have also loaded (and want to // preserve checksums for) additional entities from compatRS, which are // only needed for compatibility with ld.TidyCompatibleVersion. - modfetch.WriteGoSum(keep) + if err := modfetch.WriteGoSum(keep, mustHaveCompleteRequirements()); err != nil { + base.Fatalf("go: %v", err) + } } } diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index f5332ef52f0..79126a46b52 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -532,7 +532,7 @@ func goModSummary(m module.Version) (*modFileSummary, error) { } actual := resolveReplacement(m) - if HasModRoot() && cfg.BuildMod == "readonly" && actual.Version != "" { + if HasModRoot() && cfg.BuildMod == "readonly" && !inWorkspaceMode() && actual.Version != "" { key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} if !modfetch.HaveSum(key) { suggestion := fmt.Sprintf("; to add it:\n\tgo mod download %s", m.Path) diff --git a/src/cmd/go/testdata/script/work.txt b/src/cmd/go/testdata/script/work.txt index c68ca89a767..0d820fffc2d 100644 --- a/src/cmd/go/testdata/script/work.txt +++ b/src/cmd/go/testdata/script/work.txt @@ -1,13 +1,20 @@ go mod initwork ./a ./b cmp go.work go.work.want +! go run example.com/b +stderr 'a(\\|/)a.go:4:8: no required module provides package rsc.io/quote; to add it:\n\tgo get rsc.io/quote' +cd a +go get rsc.io/quote +go env GOMOD # go env GOMOD reports the module in a single module context +stdout $GOPATH(\\|/)src(\\|/)a(\\|/)go.mod +cd .. go run example.com/b -stdout 'Hello from module A' +stdout 'Hello, world.' # And try from a different directory cd c -go run example.com/b -stdout 'Hello from module A' +go run example.com/b +stdout 'Hello, world.' cd $GOPATH/src go list all # all includes both modules @@ -26,6 +33,9 @@ cp go.work.dup go.work stderr 'reading go.work: path .* appears multiple times in workspace' cp go.work.backup go.work +cp go.work.d go.work +go run example.com/d + -- go.work.dup -- go 1.17 @@ -41,6 +51,14 @@ directory ( ./a ./b ) +-- go.work.d -- +go 1.17 + +directory ( + a + b + d +) -- a/go.mod -- module example.com/a @@ -49,9 +67,10 @@ module example.com/a package a import "fmt" +import "rsc.io/quote" func HelloFromA() { - fmt.Println("Hello from module A") + fmt.Println(quote.Hello()) } -- b/go.mod -- @@ -66,8 +85,27 @@ import "example.com/a" func main() { a.HelloFromA() } +-- b/lib/hello.go -- +package lib + +import "example.com/a" + +func Hello() { + a.HelloFromA() +} -- c/README -- Create this directory so we can cd to it and make sure paths are interpreted relative to the go.work, not the cwd. +-- d/go.mod -- +module example.com/d + +-- d/main.go -- +package main + +import "example.com/b/lib" + +func main() { + lib.Hello() +} diff --git a/src/cmd/go/testdata/script/work_sum.txt b/src/cmd/go/testdata/script/work_sum.txt new file mode 100644 index 00000000000..99f66a40038 --- /dev/null +++ b/src/cmd/go/testdata/script/work_sum.txt @@ -0,0 +1,33 @@ +# Test adding sums to go.work.sum when sum isn't in go.mod. + +go run . +cmp go.work.sum want.sum + +-- want.sum -- +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:pvCbr/wm8HzDD3fVywevekufpn6tCGPY3spdHeZJEsw= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +rsc.io/quote v1.5.2 h1:3fEykkD9k7lYzXqCYrwGAf7iNhbk4yCjHmKBN9td4L0= +rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= +rsc.io/sampler v1.3.0 h1:HLGR/BgEtI3r0uymSP/nl2uPLsUnNJX8toRyhfpBTII= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +-- go.work -- +go 1.18 + +directory . +-- go.mod -- +go 1.18 + +module example.com/hi + +require "rsc.io/quote" v1.5.2 +-- main.go -- +package main + +import ( + "fmt" + "rsc.io/quote" +) + +func main() { + fmt.Println(quote.Hello()) +} \ No newline at end of file From 176baafd5b6d968fc0df25b344ffe826e47e6879 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 28 Jul 2021 12:32:08 -0400 Subject: [PATCH 11/25] [dev.cmdgo] cmd/go: sort roots when joining multiple main module roots When concatenating multiple main modules' roots we need to resort the list of main module roots. This avoids a panic from unsorted main module roots. This will get better when we have structured module roots. Change-Id: I68fed47b0f3b131ed3dadf45db3c442286a0ced0 Reviewed-on: https://go-review.googlesource.com/c/go/+/338111 Trust: Michael Matloob Run-TryBot: Michael Matloob Reviewed-by: Jay Conrod --- src/cmd/go/internal/modload/buildlist.go | 3 +++ src/cmd/go/testdata/script/work.txt | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index d2957a34c42..4fbe563cb83 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -1083,6 +1083,9 @@ func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requireme } roots = append(roots, min...) } + if MainModules.Len() > 1 { + module.Sort(roots) + } if rs.depth == eager && reflect.DeepEqual(roots, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) { // The root set is unchanged and rs was already eager, so keep rs to // preserve its cached ModuleGraph (if any). diff --git a/src/cmd/go/testdata/script/work.txt b/src/cmd/go/testdata/script/work.txt index 0d820fffc2d..eeaf92eaec4 100644 --- a/src/cmd/go/testdata/script/work.txt +++ b/src/cmd/go/testdata/script/work.txt @@ -36,6 +36,11 @@ cp go.work.backup go.work cp go.work.d go.work go run example.com/d +# Test that we don't run into "newRequirements called with unsorted roots" +# panic with unsorted main modules. +cp go.work.backwards go.work +go run example.com/d + -- go.work.dup -- go 1.17 @@ -109,3 +114,12 @@ import "example.com/b/lib" func main() { lib.Hello() } + +-- go.work.backwards -- +go 1.18 + +directory ( + d + b + a +) \ No newline at end of file From 137089ffb9c16288d2620a2c65dd56406f32eb84 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Wed, 14 Jul 2021 13:52:00 -0700 Subject: [PATCH 12/25] [dev.cmdgo] cmd/internal/str: move package from cmd/go/internal/str This will let cmd/cgo and cmd/link use this package for argument parsing. For golang/go#41400 Change-Id: I12ee21151bf3f00f3e8d427faaaab2453c823117 Reviewed-on: https://go-review.googlesource.com/c/go/+/334730 Trust: Jay Conrod Reviewed-by: Michael Matloob --- src/cmd/go/internal/base/base.go | 2 +- src/cmd/go/internal/base/flag.go | 2 +- src/cmd/go/internal/fix/fix.go | 2 +- src/cmd/go/internal/fmtcmd/fmt.go | 2 +- src/cmd/go/internal/generate/generate.go | 2 +- src/cmd/go/internal/get/get.go | 2 +- src/cmd/go/internal/list/list.go | 2 +- src/cmd/go/internal/load/flag.go | 2 +- src/cmd/go/internal/load/pkg.go | 2 +- src/cmd/go/internal/load/test.go | 2 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modfetch/codehost/codehost.go | 2 +- src/cmd/go/internal/modfetch/codehost/vcs.go | 2 +- src/cmd/go/internal/modget/query.go | 2 +- src/cmd/go/internal/modload/load.go | 2 +- src/cmd/go/internal/modload/query.go | 2 +- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 2 +- src/cmd/go/internal/vcs/vcs.go | 2 +- src/cmd/go/internal/work/buildid.go | 2 +- src/cmd/go/internal/work/exec.go | 2 +- src/cmd/go/internal/work/gc.go | 2 +- src/cmd/go/internal/work/gccgo.go | 2 +- src/cmd/{go => }/internal/str/path.go | 0 src/cmd/{go => }/internal/str/str.go | 0 src/cmd/{go => }/internal/str/str_test.go | 0 26 files changed, 23 insertions(+), 23 deletions(-) rename src/cmd/{go => }/internal/str/path.go (100%) rename src/cmd/{go => }/internal/str/str.go (100%) rename src/cmd/{go => }/internal/str/str_test.go (100%) diff --git a/src/cmd/go/internal/base/base.go b/src/cmd/go/internal/base/base.go index 954ce47a989..0144525e307 100644 --- a/src/cmd/go/internal/base/base.go +++ b/src/cmd/go/internal/base/base.go @@ -17,7 +17,7 @@ import ( "sync" "cmd/go/internal/cfg" - "cmd/go/internal/str" + "cmd/internal/str" ) // A Command is an implementation of a go command diff --git a/src/cmd/go/internal/base/flag.go b/src/cmd/go/internal/base/flag.go index 2262e2e992b..7e5121bffbe 100644 --- a/src/cmd/go/internal/base/flag.go +++ b/src/cmd/go/internal/base/flag.go @@ -9,7 +9,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/fsys" - "cmd/go/internal/str" + "cmd/internal/str" ) // A StringsFlag is a command-line flag that interprets its argument diff --git a/src/cmd/go/internal/fix/fix.go b/src/cmd/go/internal/fix/fix.go index 988d45e71cc..cc5940fccd8 100644 --- a/src/cmd/go/internal/fix/fix.go +++ b/src/cmd/go/internal/fix/fix.go @@ -10,7 +10,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/modload" - "cmd/go/internal/str" + "cmd/internal/str" "context" "fmt" "os" diff --git a/src/cmd/go/internal/fmtcmd/fmt.go b/src/cmd/go/internal/fmtcmd/fmt.go index 8a040087539..2b89a078acc 100644 --- a/src/cmd/go/internal/fmtcmd/fmt.go +++ b/src/cmd/go/internal/fmtcmd/fmt.go @@ -18,7 +18,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/modload" - "cmd/go/internal/str" + "cmd/internal/str" ) func init() { diff --git a/src/cmd/go/internal/generate/generate.go b/src/cmd/go/internal/generate/generate.go index 80ea32b4284..d7f2eb46107 100644 --- a/src/cmd/go/internal/generate/generate.go +++ b/src/cmd/go/internal/generate/generate.go @@ -25,8 +25,8 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/modload" - "cmd/go/internal/str" "cmd/go/internal/work" + "cmd/internal/str" ) var CmdGenerate = &base.Command{ diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index cc676428e24..075594b2716 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -17,10 +17,10 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/search" - "cmd/go/internal/str" "cmd/go/internal/vcs" "cmd/go/internal/web" "cmd/go/internal/work" + "cmd/internal/str" "golang.org/x/mod/module" ) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 04630dc341e..704d61e7c1a 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -23,8 +23,8 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modinfo" "cmd/go/internal/modload" - "cmd/go/internal/str" "cmd/go/internal/work" + "cmd/internal/str" ) var CmdList = &base.Command{ diff --git a/src/cmd/go/internal/load/flag.go b/src/cmd/go/internal/load/flag.go index 440cb861344..4e0cb5bc19e 100644 --- a/src/cmd/go/internal/load/flag.go +++ b/src/cmd/go/internal/load/flag.go @@ -6,7 +6,7 @@ package load import ( "cmd/go/internal/base" - "cmd/go/internal/str" + "cmd/internal/str" "fmt" "strings" ) diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index ac89127a4b1..f0613a4c0a6 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -36,8 +36,8 @@ import ( "cmd/go/internal/modload" "cmd/go/internal/par" "cmd/go/internal/search" - "cmd/go/internal/str" "cmd/go/internal/trace" + "cmd/internal/str" "cmd/internal/sys" "golang.org/x/mod/modfile" diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index c8282965669..42eefe37ba5 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -22,8 +22,8 @@ import ( "unicode/utf8" "cmd/go/internal/fsys" - "cmd/go/internal/str" "cmd/go/internal/trace" + "cmd/internal/str" ) var TestMainDeps = []string{ diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 3506655a4a5..6273afbbe63 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -24,7 +24,7 @@ import ( "cmd/go/internal/imports" "cmd/go/internal/load" "cmd/go/internal/modload" - "cmd/go/internal/str" + "cmd/internal/str" "golang.org/x/mod/module" "golang.org/x/mod/semver" diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go index 378fbae34f9..efb4b1516a2 100644 --- a/src/cmd/go/internal/modfetch/codehost/codehost.go +++ b/src/cmd/go/internal/modfetch/codehost/codehost.go @@ -21,7 +21,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/lockedfile" - "cmd/go/internal/str" + "cmd/internal/str" ) // Downloaded size limits. diff --git a/src/cmd/go/internal/modfetch/codehost/vcs.go b/src/cmd/go/internal/modfetch/codehost/vcs.go index c2cca084e30..5d810d2621c 100644 --- a/src/cmd/go/internal/modfetch/codehost/vcs.go +++ b/src/cmd/go/internal/modfetch/codehost/vcs.go @@ -20,7 +20,7 @@ import ( "cmd/go/internal/lockedfile" "cmd/go/internal/par" - "cmd/go/internal/str" + "cmd/internal/str" ) // A VCSError indicates an error using a version control system. diff --git a/src/cmd/go/internal/modget/query.go b/src/cmd/go/internal/modget/query.go index bbb364fa5ee..76041906f2f 100644 --- a/src/cmd/go/internal/modget/query.go +++ b/src/cmd/go/internal/modget/query.go @@ -14,7 +14,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/modload" "cmd/go/internal/search" - "cmd/go/internal/str" + "cmd/internal/str" "golang.org/x/mod/module" ) diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index e7b03b08e9d..a643cca0a9e 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -118,7 +118,7 @@ import ( "cmd/go/internal/mvs" "cmd/go/internal/par" "cmd/go/internal/search" - "cmd/go/internal/str" + "cmd/internal/str" "golang.org/x/mod/module" "golang.org/x/mod/semver" diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 05ef0a9c48f..ba137eda1d1 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -21,8 +21,8 @@ import ( "cmd/go/internal/imports" "cmd/go/internal/modfetch" "cmd/go/internal/search" - "cmd/go/internal/str" "cmd/go/internal/trace" + "cmd/internal/str" "golang.org/x/mod/module" "golang.org/x/mod/semver" diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index 7d9e2930aba..931fdcef8f4 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -18,8 +18,8 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/modload" - "cmd/go/internal/str" "cmd/go/internal/work" + "cmd/internal/str" ) var CmdRun = &base.Command{ diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 5fcea18caa9..5e02a397a63 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -31,9 +31,9 @@ import ( "cmd/go/internal/lockedfile" "cmd/go/internal/modload" "cmd/go/internal/search" - "cmd/go/internal/str" "cmd/go/internal/trace" "cmd/go/internal/work" + "cmd/internal/str" "cmd/internal/test2json" ) diff --git a/src/cmd/go/internal/vcs/vcs.go b/src/cmd/go/internal/vcs/vcs.go index 91485f6f745..97b2a631ae6 100644 --- a/src/cmd/go/internal/vcs/vcs.go +++ b/src/cmd/go/internal/vcs/vcs.go @@ -23,8 +23,8 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/search" - "cmd/go/internal/str" "cmd/go/internal/web" + "cmd/internal/str" "golang.org/x/mod/module" ) diff --git a/src/cmd/go/internal/work/buildid.go b/src/cmd/go/internal/work/buildid.go index 4e9189a3632..15f944d2af2 100644 --- a/src/cmd/go/internal/work/buildid.go +++ b/src/cmd/go/internal/work/buildid.go @@ -15,8 +15,8 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/fsys" - "cmd/go/internal/str" "cmd/internal/buildid" + "cmd/internal/str" ) // Build IDs diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index b506b836561..d51ec144f33 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -34,8 +34,8 @@ import ( "cmd/go/internal/fsys" "cmd/go/internal/load" "cmd/go/internal/modload" - "cmd/go/internal/str" "cmd/go/internal/trace" + "cmd/internal/str" ) // actionList returns the list of actions in the dag rooted at root diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 85da4f89f99..c5f3fc264fe 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -20,8 +20,8 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/fsys" "cmd/go/internal/load" - "cmd/go/internal/str" "cmd/internal/objabi" + "cmd/internal/str" "cmd/internal/sys" "crypto/sha1" ) diff --git a/src/cmd/go/internal/work/gccgo.go b/src/cmd/go/internal/work/gccgo.go index 1499536932d..3cb7b641833 100644 --- a/src/cmd/go/internal/work/gccgo.go +++ b/src/cmd/go/internal/work/gccgo.go @@ -16,8 +16,8 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/fsys" "cmd/go/internal/load" - "cmd/go/internal/str" "cmd/internal/pkgpath" + "cmd/internal/str" ) // The Gccgo toolchain. diff --git a/src/cmd/go/internal/str/path.go b/src/cmd/internal/str/path.go similarity index 100% rename from src/cmd/go/internal/str/path.go rename to src/cmd/internal/str/path.go diff --git a/src/cmd/go/internal/str/str.go b/src/cmd/internal/str/str.go similarity index 100% rename from src/cmd/go/internal/str/str.go rename to src/cmd/internal/str/str.go diff --git a/src/cmd/go/internal/str/str_test.go b/src/cmd/internal/str/str_test.go similarity index 100% rename from src/cmd/go/internal/str/str_test.go rename to src/cmd/internal/str/str_test.go From 3a69cef65a856afd1f8b9d5c22e6729d0f4d84ac Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Wed, 14 Jul 2021 15:37:06 -0700 Subject: [PATCH 13/25] [dev.cmdgo] cmd/internal/str: add utilities for quoting and splitting args JoinAndQuoteFields does the inverse of SplitQuotedFields: it joins a list of arguments with spaces into one string, quoting arguments that contain spaces or quotes. QuotedStringListFlag uses SplitQuotedFields and JoinAndQuoteFields together to define new flags that accept lists of arguments. For golang/go#41400 Change-Id: I4986b753cb5e6fabb5b489bf26aedab889f853f5 Reviewed-on: https://go-review.googlesource.com/c/go/+/334731 Trust: Jay Conrod Trust: Michael Matloob Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills Reviewed-by: Michael Matloob --- src/cmd/internal/str/str.go | 72 +++++++++++++++++++++++++++ src/cmd/internal/str/str_test.go | 83 +++++++++++++++++++++++++++++++- 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/src/cmd/internal/str/str.go b/src/cmd/internal/str/str.go index 9106ebf74d5..409cf8f7b4b 100644 --- a/src/cmd/internal/str/str.go +++ b/src/cmd/internal/str/str.go @@ -7,7 +7,9 @@ package str import ( "bytes" + "flag" "fmt" + "strings" "unicode" "unicode/utf8" ) @@ -153,3 +155,73 @@ func SplitQuotedFields(s string) ([]string, error) { } return f, nil } + +// JoinAndQuoteFields joins a list of arguments into a string that can be parsed +// with SplitQuotedFields. Arguments are quoted only if necessary; arguments +// without spaces or quotes are kept as-is. No argument may contain both +// single and double quotes. +func JoinAndQuoteFields(args []string) (string, error) { + var buf []byte + for i, arg := range args { + if i > 0 { + buf = append(buf, ' ') + } + var sawSpace, sawSingleQuote, sawDoubleQuote bool + for _, c := range arg { + switch { + case c > unicode.MaxASCII: + continue + case isSpaceByte(byte(c)): + sawSpace = true + case c == '\'': + sawSingleQuote = true + case c == '"': + sawDoubleQuote = true + } + } + switch { + case !sawSpace && !sawSingleQuote && !sawDoubleQuote: + buf = append(buf, []byte(arg)...) + + case !sawSingleQuote: + buf = append(buf, '\'') + buf = append(buf, []byte(arg)...) + buf = append(buf, '\'') + + case !sawDoubleQuote: + buf = append(buf, '"') + buf = append(buf, []byte(arg)...) + buf = append(buf, '"') + + default: + return "", fmt.Errorf("argument %q contains both single and double quotes and cannot be quoted", arg) + } + } + return string(buf), nil +} + +// A QuotedStringListFlag parses a list of string arguments encoded with +// JoinAndQuoteFields. It is useful for flags like cmd/link's -extldflags. +type QuotedStringListFlag []string + +var _ flag.Value = (*QuotedStringListFlag)(nil) + +func (f *QuotedStringListFlag) Set(v string) error { + fs, err := SplitQuotedFields(v) + if err != nil { + return err + } + *f = fs[:len(fs):len(fs)] + return nil +} + +func (f *QuotedStringListFlag) String() string { + if f == nil { + return "" + } + s, err := JoinAndQuoteFields(*f) + if err != nil { + return strings.Join(*f, " ") + } + return s +} diff --git a/src/cmd/internal/str/str_test.go b/src/cmd/internal/str/str_test.go index 147ce1a63ef..3609af6a06d 100644 --- a/src/cmd/internal/str/str_test.go +++ b/src/cmd/internal/str/str_test.go @@ -4,7 +4,11 @@ package str -import "testing" +import ( + "reflect" + "strings" + "testing" +) var foldDupTests = []struct { list []string @@ -25,3 +29,80 @@ func TestFoldDup(t *testing.T) { } } } + +func TestSplitQuotedFields(t *testing.T) { + for _, test := range []struct { + name string + value string + want []string + wantErr string + }{ + {name: "empty", value: "", want: nil}, + {name: "space", value: " ", want: nil}, + {name: "one", value: "a", want: []string{"a"}}, + {name: "leading_space", value: " a", want: []string{"a"}}, + {name: "trailing_space", value: "a ", want: []string{"a"}}, + {name: "two", value: "a b", want: []string{"a", "b"}}, + {name: "two_multi_space", value: "a b", want: []string{"a", "b"}}, + {name: "two_tab", value: "a\tb", want: []string{"a", "b"}}, + {name: "two_newline", value: "a\nb", want: []string{"a", "b"}}, + {name: "quote_single", value: `'a b'`, want: []string{"a b"}}, + {name: "quote_double", value: `"a b"`, want: []string{"a b"}}, + {name: "quote_both", value: `'a '"b "`, want: []string{"a ", "b "}}, + {name: "quote_contains", value: `'a "'"'b"`, want: []string{`a "`, `'b`}}, + {name: "escape", value: `\'`, want: []string{`\'`}}, + {name: "quote_unclosed", value: `'a`, wantErr: "unterminated ' string"}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := SplitQuotedFields(test.value) + if err != nil { + if test.wantErr == "" { + t.Fatalf("unexpected error: %v", err) + } else if errMsg := err.Error(); !strings.Contains(errMsg, test.wantErr) { + t.Fatalf("error %q does not contain %q", errMsg, test.wantErr) + } + return + } + if test.wantErr != "" { + t.Fatalf("unexpected success; wanted error containing %q", test.wantErr) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("got %q; want %q", got, test.want) + } + }) + } +} + +func TestJoinAndQuoteFields(t *testing.T) { + for _, test := range []struct { + name string + args []string + want, wantErr string + }{ + {name: "empty", args: nil, want: ""}, + {name: "one", args: []string{"a"}, want: "a"}, + {name: "two", args: []string{"a", "b"}, want: "a b"}, + {name: "space", args: []string{"a ", "b"}, want: "'a ' b"}, + {name: "newline", args: []string{"a\n", "b"}, want: "'a\n' b"}, + {name: "quote", args: []string{`'a `, "b"}, want: `"'a " b`}, + {name: "unquoteable", args: []string{`'"`}, wantErr: "contains both single and double quotes and cannot be quoted"}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := JoinAndQuoteFields(test.args) + if err != nil { + if test.wantErr == "" { + t.Fatalf("unexpected error: %v", err) + } else if errMsg := err.Error(); !strings.Contains(errMsg, test.wantErr) { + t.Fatalf("error %q does not contain %q", errMsg, test.wantErr) + } + return + } + if test.wantErr != "" { + t.Fatalf("unexpected success; wanted error containing %q", test.wantErr) + } + if got != test.want { + t.Errorf("got %s; want %s", got, test.want) + } + }) + } +} From 52e970b1c86f18806232adb0e1f42636645d21ff Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Wed, 14 Jul 2021 16:57:24 -0700 Subject: [PATCH 14/25] [dev.cmdgo] cmd: support space and quotes in CC and CXX The CC and CXX environment variables now support spaces and quotes (both double and single). This fixes two issues: first, if CC is a single path that contains spaces (like 'c:\Program Files\gcc\bin\gcc.exe'), that should now work if the space is quoted or escaped (#41400). Second, if CC or CXX has multiple arguments (like 'gcc -O2'), they are now split correctly, and the arguments are passed before other arguments when invoking the C compiler. Previously, strings.Fields was used to split arguments, and the arguments were placed later in the command line. (#43078). Fixes golang/go#41400 Fixes golang/go#43078 Change-Id: I2d5d89ddb19c94adef65982a8137b01f037d5c11 Reviewed-on: https://go-review.googlesource.com/c/go/+/334732 Trust: Jay Conrod Trust: Michael Matloob Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Michael Matloob --- src/cmd/cgo/gcc.go | 46 ++++++++++----- src/cmd/cgo/main.go | 8 +-- .../compile/internal/ssa/stmtlines_test.go | 7 ++- src/cmd/dist/buildtool.go | 1 + src/cmd/go/internal/envcmd/env.go | 34 +++++++---- src/cmd/go/internal/work/exec.go | 43 +++++--------- src/cmd/go/internal/work/gc.go | 37 +++++------- src/cmd/go/internal/work/init.go | 16 +++++- src/cmd/go/script_test.go | 1 + .../testdata/script/cgo_path_space_quote.txt | 56 +++++++++++++++++++ src/cmd/internal/dwarf/dwarf.go | 6 +- src/cmd/link/dwarf_test.go | 8 ++- src/cmd/link/internal/ld/lib.go | 39 ++++++++----- src/cmd/link/internal/ld/main.go | 7 ++- 14 files changed, 206 insertions(+), 103 deletions(-) create mode 100644 src/cmd/go/testdata/script/cgo_path_space_quote.txt diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index a73e998877a..92adb1ed9cc 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -23,10 +23,13 @@ import ( "internal/xcoff" "math" "os" + "os/exec" "strconv" "strings" "unicode" "unicode/utf8" + + "cmd/internal/str" ) var debugDefine = flag.Bool("debug-define", false, "print relevant #defines") @@ -382,7 +385,7 @@ func (p *Package) guessKinds(f *File) []*Name { stderr = p.gccErrors(b.Bytes()) } if stderr == "" { - fatalf("%s produced no output\non input:\n%s", p.gccBaseCmd()[0], b.Bytes()) + fatalf("%s produced no output\non input:\n%s", gccBaseCmd[0], b.Bytes()) } completed := false @@ -457,7 +460,7 @@ func (p *Package) guessKinds(f *File) []*Name { } if !completed { - fatalf("%s did not produce error at completed:1\non input:\n%s\nfull error output:\n%s", p.gccBaseCmd()[0], b.Bytes(), stderr) + fatalf("%s did not produce error at completed:1\non input:\n%s\nfull error output:\n%s", gccBaseCmd[0], b.Bytes(), stderr) } for i, n := range names { @@ -488,7 +491,7 @@ func (p *Package) guessKinds(f *File) []*Name { // to users debugging preamble mistakes. See issue 8442. preambleErrors := p.gccErrors([]byte(f.Preamble)) if len(preambleErrors) > 0 { - error_(token.NoPos, "\n%s errors for preamble:\n%s", p.gccBaseCmd()[0], preambleErrors) + error_(token.NoPos, "\n%s errors for preamble:\n%s", gccBaseCmd[0], preambleErrors) } fatalf("unresolved names") @@ -1545,20 +1548,37 @@ func gofmtPos(n ast.Expr, pos token.Pos) string { return fmt.Sprintf("/*line :%d:%d*/%s", p.Line, p.Column, s) } -// gccBaseCmd returns the start of the compiler command line. +// checkGCCBaseCmd returns the start of the compiler command line. // It uses $CC if set, or else $GCC, or else the compiler recorded // during the initial build as defaultCC. // defaultCC is defined in zdefaultcc.go, written by cmd/dist. -func (p *Package) gccBaseCmd() []string { +// +// The compiler command line is split into arguments on whitespace. Quotes +// are understood, so arguments may contain whitespace. +// +// checkGCCBaseCmd confirms that the compiler exists in PATH, returning +// an error if it does not. +func checkGCCBaseCmd() ([]string, error) { // Use $CC if set, since that's what the build uses. - if ret := strings.Fields(os.Getenv("CC")); len(ret) > 0 { - return ret + value := os.Getenv("CC") + if value == "" { + // Try $GCC if set, since that's what we used to use. + value = os.Getenv("GCC") } - // Try $GCC if set, since that's what we used to use. - if ret := strings.Fields(os.Getenv("GCC")); len(ret) > 0 { - return ret + if value == "" { + value = defaultCC(goos, goarch) } - return strings.Fields(defaultCC(goos, goarch)) + args, err := str.SplitQuotedFields(value) + if err != nil { + return nil, err + } + if len(args) == 0 { + return nil, errors.New("CC not set and no default found") + } + if _, err := exec.LookPath(args[0]); err != nil { + return nil, fmt.Errorf("C compiler %q not found: %v", args[0], err) + } + return args[:len(args):len(args)], nil } // gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm". @@ -1604,7 +1624,7 @@ func gccTmp() string { // gccCmd returns the gcc command line to use for compiling // the input. func (p *Package) gccCmd() []string { - c := append(p.gccBaseCmd(), + c := append(gccBaseCmd, "-w", // no warnings "-Wno-error", // warnings are not errors "-o"+gccTmp(), // write object to tmp @@ -2005,7 +2025,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6 // #defines that gcc encountered while processing the input // and its included files. func (p *Package) gccDefines(stdin []byte) string { - base := append(p.gccBaseCmd(), "-E", "-dM", "-xc") + base := append(gccBaseCmd, "-E", "-dM", "-xc") base = append(base, p.gccMachine()...) stdout, _ := runGcc(stdin, append(append(base, p.GccOptions...), "-")) return stdout diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go index c6a0c525e64..14642b7576b 100644 --- a/src/cmd/cgo/main.go +++ b/src/cmd/cgo/main.go @@ -21,7 +21,6 @@ import ( "io" "io/ioutil" "os" - "os/exec" "path/filepath" "reflect" "runtime" @@ -248,6 +247,7 @@ var importSyscall = flag.Bool("import_syscall", true, "import syscall in generat var trimpath = flag.String("trimpath", "", "applies supplied rewrites or trims prefixes to recorded source file paths") var goarch, goos, gomips, gomips64 string +var gccBaseCmd []string func main() { objabi.AddVersionFlag() // -V @@ -305,10 +305,10 @@ func main() { p := newPackage(args[:i]) // We need a C compiler to be available. Check this. - gccName := p.gccBaseCmd()[0] - _, err := exec.LookPath(gccName) + var err error + gccBaseCmd, err = checkGCCBaseCmd() if err != nil { - fatalf("C compiler %q not found: %v", gccName, err) + fatalf("%v", err) os.Exit(2) } diff --git a/src/cmd/compile/internal/ssa/stmtlines_test.go b/src/cmd/compile/internal/ssa/stmtlines_test.go index a510d0b3d06..843db8c07ef 100644 --- a/src/cmd/compile/internal/ssa/stmtlines_test.go +++ b/src/cmd/compile/internal/ssa/stmtlines_test.go @@ -2,6 +2,7 @@ package ssa_test import ( cmddwarf "cmd/internal/dwarf" + "cmd/internal/str" "debug/dwarf" "debug/elf" "debug/macho" @@ -57,7 +58,11 @@ func TestStmtLines(t *testing.T) { if extld == "" { extld = "gcc" } - enabled, err := cmddwarf.IsDWARFEnabledOnAIXLd(extld) + extldArgs, err := str.SplitQuotedFields(extld) + if err != nil { + t.Fatal(err) + } + enabled, err := cmddwarf.IsDWARFEnabledOnAIXLd(extldArgs) if err != nil { t.Fatal(err) } diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go index 26b33e389fe..320c62f8505 100644 --- a/src/cmd/dist/buildtool.go +++ b/src/cmd/dist/buildtool.go @@ -47,6 +47,7 @@ var bootstrapDirs = []string{ "cmd/internal/objabi", "cmd/internal/pkgpath", "cmd/internal/src", + "cmd/internal/str", "cmd/internal/sys", "cmd/link", "cmd/link/internal/...", diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index f68090f21f3..5c45e343309 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -26,6 +26,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/work" + "cmd/internal/str" ) var CmdEnv = &base.Command{ @@ -104,13 +105,13 @@ func MkEnv() []cfg.EnvVar { env = append(env, cfg.EnvVar{Name: key, Value: val}) } - cc := cfg.DefaultCC(cfg.Goos, cfg.Goarch) - if env := strings.Fields(cfg.Getenv("CC")); len(env) > 0 { - cc = env[0] + cc := cfg.Getenv("CC") + if cc == "" { + cc = cfg.DefaultCC(cfg.Goos, cfg.Goarch) } - cxx := cfg.DefaultCXX(cfg.Goos, cfg.Goarch) - if env := strings.Fields(cfg.Getenv("CXX")); len(env) > 0 { - cxx = env[0] + cxx := cfg.Getenv("CXX") + if cxx == "" { + cxx = cfg.DefaultCXX(cfg.Goos, cfg.Goarch) } env = append(env, cfg.EnvVar{Name: "AR", Value: envOr("AR", "ar")}) env = append(env, cfg.EnvVar{Name: "CC", Value: cc}) @@ -458,10 +459,23 @@ func checkEnvWrite(key, val string) error { if !filepath.IsAbs(val) && val != "" { return fmt.Errorf("GOPATH entry is relative; must be absolute path: %q", val) } - // Make sure CC and CXX are absolute paths - case "CC", "CXX", "GOMODCACHE": - if !filepath.IsAbs(val) && val != "" && val != filepath.Base(val) { - return fmt.Errorf("%s entry is relative; must be absolute path: %q", key, val) + case "GOMODCACHE": + if !filepath.IsAbs(val) && val != "" { + return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q", val) + } + case "CC", "CXX": + if val == "" { + break + } + args, err := str.SplitQuotedFields(val) + if err != nil { + return fmt.Errorf("invalid %s: %v", key, err) + } + if len(args) == 0 { + return fmt.Errorf("%s entry cannot contain only space", key) + } + if !filepath.IsAbs(args[0]) && args[0] != filepath.Base(args[0]) { + return fmt.Errorf("%s entry is relative; must be absolute path: %q", key, args[0]) } } diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index d51ec144f33..6c646ebf28d 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -1480,6 +1480,8 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, return nil, nil, errPrintedOutput } if len(out) > 0 { + // NOTE: we don't attempt to parse quotes and unescapes here. pkg-config + // is typically used within shell backticks, which treats quotes literally. ldflags = strings.Fields(string(out)) if err := checkLinkerFlags("LDFLAGS", "pkg-config --libs", ldflags); err != nil { return nil, nil, err @@ -2422,12 +2424,6 @@ func (b *Builder) gccld(a *Action, p *load.Package, objdir, outfile string, flag return err } -// Grab these before main helpfully overwrites them. -var ( - origCC = cfg.Getenv("CC") - origCXX = cfg.Getenv("CXX") -) - // gccCmd returns a gcc command line prefix // defaultCC is defined in zdefaultcc.go, written by cmd/dist. func (b *Builder) GccCmd(incdir, workdir string) []string { @@ -2447,40 +2443,23 @@ func (b *Builder) gfortranCmd(incdir, workdir string) []string { // ccExe returns the CC compiler setting without all the extra flags we add implicitly. func (b *Builder) ccExe() []string { - return b.compilerExe(origCC, cfg.DefaultCC(cfg.Goos, cfg.Goarch)) + return envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) } // cxxExe returns the CXX compiler setting without all the extra flags we add implicitly. func (b *Builder) cxxExe() []string { - return b.compilerExe(origCXX, cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) + return envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) } // fcExe returns the FC compiler setting without all the extra flags we add implicitly. func (b *Builder) fcExe() []string { - return b.compilerExe(cfg.Getenv("FC"), "gfortran") -} - -// compilerExe returns the compiler to use given an -// environment variable setting (the value not the name) -// and a default. The resulting slice is usually just the name -// of the compiler but can have additional arguments if they -// were present in the environment value. -// For example if CC="gcc -DGOPHER" then the result is ["gcc", "-DGOPHER"]. -func (b *Builder) compilerExe(envValue string, def string) []string { - compiler := strings.Fields(envValue) - if len(compiler) == 0 { - compiler = strings.Fields(def) - } - return compiler + return envList("FC", "gfortran") } // compilerCmd returns a command line prefix for the given environment // variable and using the default command when the variable is empty. func (b *Builder) compilerCmd(compiler []string, incdir, workdir string) []string { - // NOTE: env.go's mkEnv knows that the first three - // strings returned are "gcc", "-I", incdir (and cuts them off). - a := []string{compiler[0], "-I", incdir} - a = append(a, compiler[1:]...) + a := append(compiler, "-I", incdir) // Definitely want -fPIC but on Windows gcc complains // "-fPIC ignored for target (all code is position independent)" @@ -2651,12 +2630,20 @@ func (b *Builder) gccArchArgs() []string { // envList returns the value of the given environment variable broken // into fields, using the default value when the variable is empty. +// +// The environment variable must be quoted correctly for +// str.SplitQuotedFields. This should be done before building +// anything, for example, in BuildInit. func envList(key, def string) []string { v := cfg.Getenv(key) if v == "" { v = def } - return strings.Fields(v) + args, err := str.SplitQuotedFields(v) + if err != nil { + panic(fmt.Sprintf("could not parse environment variable %s with value %q: %v", key, v, err)) + } + return args } // CFlags returns the flags to use when invoking the C, C++ or Fortran compilers, or cgo. diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index c5f3fc264fe..1fc825de47d 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -536,33 +536,18 @@ func packInternal(afile string, ofiles []string) error { } // setextld sets the appropriate linker flags for the specified compiler. -func setextld(ldflags []string, compiler []string) []string { +func setextld(ldflags []string, compiler []string) ([]string, error) { for _, f := range ldflags { if f == "-extld" || strings.HasPrefix(f, "-extld=") { // don't override -extld if supplied - return ldflags + return ldflags, nil } } - ldflags = append(ldflags, "-extld="+compiler[0]) - if len(compiler) > 1 { - extldflags := false - add := strings.Join(compiler[1:], " ") - for i, f := range ldflags { - if f == "-extldflags" && i+1 < len(ldflags) { - ldflags[i+1] = add + " " + ldflags[i+1] - extldflags = true - break - } else if strings.HasPrefix(f, "-extldflags=") { - ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] - extldflags = true - break - } - } - if !extldflags { - ldflags = append(ldflags, "-extldflags="+add) - } + joined, err := str.JoinAndQuoteFields(compiler) + if err != nil { + return nil, err } - return ldflags + return append(ldflags, "-extld="+joined), nil } // pluginPath computes the package path for a plugin main package. @@ -649,7 +634,10 @@ func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) } ldflags = append(ldflags, forcedLdflags...) ldflags = append(ldflags, root.Package.Internal.Ldflags...) - ldflags = setextld(ldflags, compiler) + ldflags, err := setextld(ldflags, compiler) + if err != nil { + return err + } // On OS X when using external linking to build a shared library, // the argument passed here to -o ends up recorded in the final @@ -693,7 +681,10 @@ func (gcToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, } else { compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) } - ldflags = setextld(ldflags, compiler) + ldflags, err := setextld(ldflags, compiler) + if err != nil { + return err + } for _, d := range toplevelactions { if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries continue diff --git a/src/cmd/go/internal/work/init.go b/src/cmd/go/internal/work/init.go index 37a3e2d0ffd..022137390f8 100644 --- a/src/cmd/go/internal/work/init.go +++ b/src/cmd/go/internal/work/init.go @@ -11,6 +11,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/fsys" "cmd/go/internal/modload" + "cmd/internal/str" "cmd/internal/sys" "flag" "fmt" @@ -39,9 +40,18 @@ func BuildInit() { cfg.BuildPkgdir = p } - // Make sure CC and CXX are absolute paths - for _, key := range []string{"CC", "CXX"} { - if path := cfg.Getenv(key); !filepath.IsAbs(path) && path != "" && path != filepath.Base(path) { + // Make sure CC, CXX, and FC are absolute paths. + for _, key := range []string{"CC", "CXX", "FC"} { + value := cfg.Getenv(key) + args, err := str.SplitQuotedFields(value) + if err != nil { + base.Fatalf("go %s: %s environment variable could not be parsed: %v", flag.Args()[0], key, err) + } + if len(args) == 0 { + continue + } + path := args[0] + if !filepath.IsAbs(path) && path != filepath.Base(path) { base.Fatalf("go %s: %s environment variable is relative; must be absolute path: %s\n", flag.Args()[0], key, path) } } diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index 9ca297e89b4..8a7c77a46fa 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -184,6 +184,7 @@ func (ts *testScript) setup() { "devnull=" + os.DevNull, "goversion=" + goVersion(ts), ":=" + string(os.PathListSeparator), + "/=" + string(os.PathSeparator), } if !testenv.HasExternalNetwork() { ts.env = append(ts.env, "TESTGONETWORK=panic", "TESTGOVCS=panic") diff --git a/src/cmd/go/testdata/script/cgo_path_space_quote.txt b/src/cmd/go/testdata/script/cgo_path_space_quote.txt new file mode 100644 index 00000000000..3b89bfb8001 --- /dev/null +++ b/src/cmd/go/testdata/script/cgo_path_space_quote.txt @@ -0,0 +1,56 @@ +# This test checks that the CC environment variable may contain quotes and +# spaces. Arguments are normally split on spaces, tabs, newlines. If an +# argument contains these characters, the entire argument may be quoted +# with single or double quotes. This is the same as -gcflags and similar +# options. + +[short] skip +[!exec:clang] [!exec:gcc] skip + +env GOENV=$WORK/go.env +mkdir 'program files' +go build -o 'program files' './which cc/which cc.go' +[exec:clang] env CC='"'$PWD${/}program' 'files${/}which' 'cc"' 'clang +[!exec:clang] env CC='"'$PWD${/}program' 'files${/}which' 'cc"' 'gcc +go env CC +stdout 'program files[/\\]which cc" (clang|gcc)$' +go env -w CC=$CC +env CC= +go env CC +stdout 'program files[/\\]which cc" (clang|gcc)$' + +go run . + +-- go.mod -- +module test + +go 1.17 +-- which cc/which cc.go -- +package main + +import ( + "fmt" + "os" + "os/exec" +) + +func main() { + args := append([]string{"-DWRAPPER_WAS_USED=1"}, os.Args[2:]...) + cmd := exec.Command(os.Args[1], args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} +-- hello.go -- +package main + +// int x = WRAPPER_WAS_USED; +import "C" +import "fmt" + +func main() { + fmt.Println(C.x) +} diff --git a/src/cmd/internal/dwarf/dwarf.go b/src/cmd/internal/dwarf/dwarf.go index ec441c2bcb6..54c4c4d56d7 100644 --- a/src/cmd/internal/dwarf/dwarf.go +++ b/src/cmd/internal/dwarf/dwarf.go @@ -1620,8 +1620,10 @@ func (s byChildIndex) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // current extld. // AIX ld doesn't support DWARF with -bnoobjreorder with version // prior to 7.2.2. -func IsDWARFEnabledOnAIXLd(extld string) (bool, error) { - out, err := exec.Command(extld, "-Wl,-V").CombinedOutput() +func IsDWARFEnabledOnAIXLd(extld []string) (bool, error) { + name, args := extld[0], extld[1:] + args = append(args, "-Wl,-V") + out, err := exec.Command(name, args...).CombinedOutput() if err != nil { // The normal output should display ld version and // then fails because ".main" is not defined: diff --git a/src/cmd/link/dwarf_test.go b/src/cmd/link/dwarf_test.go index 3ca59bd47f0..f7bbb014d9e 100644 --- a/src/cmd/link/dwarf_test.go +++ b/src/cmd/link/dwarf_test.go @@ -8,6 +8,7 @@ import ( "bytes" cmddwarf "cmd/internal/dwarf" "cmd/internal/objfile" + "cmd/internal/str" "debug/dwarf" "internal/testenv" "os" @@ -67,8 +68,11 @@ func testDWARF(t *testing.T, buildmode string, expectDWARF bool, env ...string) if extld == "" { extld = "gcc" } - var err error - expectDWARF, err = cmddwarf.IsDWARFEnabledOnAIXLd(extld) + extldArgs, err := str.SplitQuotedFields(extld) + if err != nil { + t.Fatal(err) + } + expectDWARF, err = cmddwarf.IsDWARFEnabledOnAIXLd(extldArgs) if err != nil { t.Fatal(err) } diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go index 644faeb2fbc..4cfee4a1e77 100644 --- a/src/cmd/link/internal/ld/lib.go +++ b/src/cmd/link/internal/ld/lib.go @@ -464,23 +464,24 @@ func loadinternal(ctxt *Link, name string) *sym.Library { } // extld returns the current external linker. -func (ctxt *Link) extld() string { - if *flagExtld == "" { - *flagExtld = "gcc" +func (ctxt *Link) extld() []string { + if len(flagExtld) == 0 { + flagExtld = []string{"gcc"} } - return *flagExtld + return flagExtld } // findLibPathCmd uses cmd command to find gcc library libname. // It returns library full path if found, or "none" if not found. func (ctxt *Link) findLibPathCmd(cmd, libname string) string { extld := ctxt.extld() - args := hostlinkArchArgs(ctxt.Arch) + name, args := extld[0], extld[1:] + args = append(args, hostlinkArchArgs(ctxt.Arch)...) args = append(args, cmd) if ctxt.Debugvlog != 0 { ctxt.Logf("%s %v\n", extld, args) } - out, err := exec.Command(extld, args...).Output() + out, err := exec.Command(name, args...).Output() if err != nil { if ctxt.Debugvlog != 0 { ctxt.Logf("not using a %s file because compiler failed\n%v\n%s\n", libname, err, out) @@ -1240,7 +1241,7 @@ func (ctxt *Link) hostlink() { } var argv []string - argv = append(argv, ctxt.extld()) + argv = append(argv, ctxt.extld()...) argv = append(argv, hostlinkArchArgs(ctxt.Arch)...) if *FlagS || debug_s { @@ -1401,7 +1402,9 @@ func (ctxt *Link) hostlink() { // If gold is not installed, gcc will silently switch // back to ld.bfd. So we parse the version information // and provide a useful error if gold is missing. - cmd := exec.Command(*flagExtld, "-fuse-ld=gold", "-Wl,--version") + name, args := flagExtld[0], flagExtld[1:] + args = append(args, "-fuse-ld=gold", "-Wl,--version") + cmd := exec.Command(name, args...) if out, err := cmd.CombinedOutput(); err == nil { if !bytes.Contains(out, []byte("GNU gold")) { log.Fatalf("ARM external linker must be gold (issue #15696), but is not: %s", out) @@ -1414,7 +1417,9 @@ func (ctxt *Link) hostlink() { altLinker = "bfd" // Provide a useful error if ld.bfd is missing. - cmd := exec.Command(*flagExtld, "-fuse-ld=bfd", "-Wl,--version") + name, args := flagExtld[0], flagExtld[1:] + args = append(args, "-fuse-ld=bfd", "-Wl,--version") + cmd := exec.Command(name, args...) if out, err := cmd.CombinedOutput(); err == nil { if !bytes.Contains(out, []byte("GNU ld")) { log.Fatalf("ARM64 external linker must be ld.bfd (issue #35197), please install devel/binutils") @@ -1482,10 +1487,11 @@ func (ctxt *Link) hostlink() { argv = append(argv, "/lib/crt0_64.o") extld := ctxt.extld() + name, args := extld[0], extld[1:] // Get starting files. getPathFile := func(file string) string { - args := []string{"-maix64", "--print-file-name=" + file} - out, err := exec.Command(extld, args...).CombinedOutput() + args := append(args, "-maix64", "--print-file-name="+file) + out, err := exec.Command(name, args...).CombinedOutput() if err != nil { log.Fatalf("running %s failed: %v\n%s", extld, err, out) } @@ -1567,14 +1573,18 @@ func (ctxt *Link) hostlink() { } } - for _, p := range strings.Fields(*flagExtldflags) { + for _, p := range flagExtldflags { argv = append(argv, p) checkStatic(p) } if ctxt.HeadType == objabi.Hwindows { // Determine which linker we're using. Add in the extldflags in // case used has specified "-fuse-ld=...". - cmd := exec.Command(*flagExtld, *flagExtldflags, "-Wl,--version") + extld := ctxt.extld() + name, args := extld[0], extld[1:] + args = append(args, flagExtldflags...) + args = append(args, "-Wl,--version") + cmd := exec.Command(name, args...) usingLLD := false if out, err := cmd.CombinedOutput(); err == nil { if bytes.Contains(out, []byte("LLD ")) { @@ -1718,8 +1728,7 @@ func linkerFlagSupported(arch *sys.Arch, linker, altLinker, flag string) bool { flags := hostlinkArchArgs(arch) keep := false skip := false - extldflags := strings.Fields(*flagExtldflags) - for _, f := range append(extldflags, ldflag...) { + for _, f := range append(flagExtldflags, ldflag...) { if keep { flags = append(flags, f) keep = false diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index cba0e3d81fe..33b03b50248 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -34,6 +34,7 @@ import ( "bufio" "cmd/internal/goobj" "cmd/internal/objabi" + "cmd/internal/str" "cmd/internal/sys" "cmd/link/internal/benchmark" "flag" @@ -53,6 +54,8 @@ var ( func init() { flag.Var(&rpath, "r", "set the ELF dynamic linker search `path` to dir1:dir2:...") + flag.Var(&flagExtld, "extld", "use `linker` when linking in external mode") + flag.Var(&flagExtldflags, "extldflags", "pass `flags` to external linker") } // Flags used by the linker. The exported flags are used by the architecture-specific packages. @@ -72,8 +75,8 @@ var ( flagLibGCC = flag.String("libgcc", "", "compiler support lib for internal linking; use \"none\" to disable") flagTmpdir = flag.String("tmpdir", "", "use `directory` for temporary files") - flagExtld = flag.String("extld", "", "use `linker` when linking in external mode") - flagExtldflags = flag.String("extldflags", "", "pass `flags` to external linker") + flagExtld str.QuotedStringListFlag + flagExtldflags str.QuotedStringListFlag flagExtar = flag.String("extar", "", "archive program for buildmode=c-archive") flagA = flag.Bool("a", false, "no-op (deprecated)") From 90830699aee61a154e989b2d9f8ce3ff4eabbce1 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Fri, 30 Jul 2021 14:10:25 -0400 Subject: [PATCH 15/25] [dev.cmdgo] cmd/go: allow expliticly setting -mod=readonly in workspace mode Change-Id: Iedbe47d087d17984a9d839c13c4b7e6c1fa0deaa Reviewed-on: https://go-review.googlesource.com/c/go/+/338594 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modload/init.go | 4 ++-- src/cmd/go/testdata/script/work.txt | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index a3337d6d23a..53c73cb4a08 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -947,8 +947,8 @@ func requirementsFromModFiles(ctx context.Context, modFiles []*modfile.File) *Re // wasn't provided. setDefaultBuildMod may be called multiple times. func setDefaultBuildMod() { if cfg.BuildModExplicit { - if inWorkspaceMode() { - base.Fatalf("go: -mod can't be set explicitly when in workspace mode." + + if inWorkspaceMode() && cfg.BuildMod != "readonly" { + base.Fatalf("go: -mod may only be set to readonly when in workspace mode." + "\n\tRemove the -mod flag to use the default readonly value," + "\n\tor set -workfile=off to disable workspace mode.") } diff --git a/src/cmd/go/testdata/script/work.txt b/src/cmd/go/testdata/script/work.txt index eeaf92eaec4..bcbabbacef8 100644 --- a/src/cmd/go/testdata/script/work.txt +++ b/src/cmd/go/testdata/script/work.txt @@ -21,10 +21,11 @@ go list all # all includes both modules stdout 'example.com/a' stdout 'example.com/b' -# -mod can't be set in workspace mode, even to readonly -! go list -mod=readonly all -stderr '^go: -mod can''t be set explicitly' -go list -mod=readonly -workfile=off all +# -mod can only be set to readonly in workspace mode +go list -mod=readonly all +! go list -mod=mod all +stderr '^go: -mod may only be set to readonly when in workspace mode' +go list -mod=mod -workfile=off all # Test that duplicates in the directory list return an error cp go.work go.work.backup From 47694b59eb30bfe6a1c12a2eaaf631a4e956b9c7 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 28 Jun 2021 15:48:03 -0400 Subject: [PATCH 16/25] [dev.cmdgo] cmd/go: provide a more helpful missing required module error in workspaces If the user is in a workspace, they might not be in the main module they need to run go get from to add a module that provides a missing dependency. Figure out what that module is from the import stack (there might be multiple but we pick according to the stack computed by the loader for errors) and tell the user to cd to that directory first in the message. Change-Id: I7c919eb61ea3dd122334ff1acd2d7e817cad4b25 Reviewed-on: https://go-review.googlesource.com/c/go/+/334940 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modload/import.go | 5 +++- src/cmd/go/internal/modload/init.go | 33 +++++++++++++++++++-------- src/cmd/go/internal/modload/load.go | 9 ++++++++ src/cmd/go/testdata/script/work.txt | 2 +- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 773d8b600b2..088d0c14ecb 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -32,7 +32,7 @@ type ImportMissingError struct { Module module.Version QueryErr error - ImportingModule module.Version + ImportingMainModule module.Version // isStd indicates whether we would expect to find the package in the standard // library. This is normally true for all dotless import paths, but replace @@ -73,6 +73,9 @@ func (e *ImportMissingError) Error() string { if e.QueryErr != nil { return fmt.Sprintf("%s: %v", message, e.QueryErr) } + if e.ImportingMainModule.Path != "" && e.ImportingMainModule != MainModules.ModContainingCWD() { + return fmt.Sprintf("%s; to add it:\n\tcd %s\n\tgo get %s", message, MainModules.ModRoot(e.ImportingMainModule), e.Path) + } return fmt.Sprintf("%s; to add it:\n\tgo get %s", message, e.Path) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 53c73cb4a08..18b07cb1257 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -89,6 +89,8 @@ type MainModuleSet struct { modFiles map[module.Version]*modfile.File + modContainingCWD module.Version + indexMu sync.Mutex indices map[module.Version]*modFileIndex } @@ -184,6 +186,13 @@ func (mms *MainModuleSet) Len() int { return len(mms.versions) } +// ModContainingCWD returns the main module containing the working directory, +// or module.Version{} if none of the main modules contain the working +// directory. +func (mms *MainModuleSet) ModContainingCWD() module.Version { + return mms.modContainingCWD +} + var MainModules *MainModuleSet type Root int @@ -315,8 +324,7 @@ func Init() { } else if inWorkspaceMode() { // We're in workspace mode. } else { - modRoots = findModuleRoots(base.Cwd()) - if modRoots == nil { + if modRoot := findModuleRoot(base.Cwd()); modRoot == "" { if cfg.ModFile != "" { base.Fatalf("go: cannot find main module, but -modfile was set.\n\t-modfile cannot be used to set the module root directory.") } @@ -328,17 +336,18 @@ func Init() { // Stay in GOPATH mode. return } - } else if search.InDir(modRoots[0], os.TempDir()) == "." { + } else if search.InDir(modRoot, os.TempDir()) == "." { // If you create /tmp/go.mod for experimenting, // then any tests that create work directories under /tmp // will find it and get modules when they're not expecting them. // It's a bit of a peculiar thing to disallow but quite mysterious // when it happens. See golang.org/issue/26708. - modRoots = nil fmt.Fprintf(os.Stderr, "go: warning: ignoring go.mod in system temp root %v\n", os.TempDir()) if !mustUseModules { return } + } else { + modRoots = []string{modRoot} } } if cfg.ModFile != "" && !strings.HasSuffix(cfg.ModFile, ".mod") { @@ -424,12 +433,11 @@ func WillBeEnabled() bool { return false } - if modRoots := findModuleRoots(base.Cwd()); modRoots == nil { + if modRoot := findModuleRoot(base.Cwd()); modRoot == "" { // GO111MODULE is 'auto', and we can't find a module root. // Stay in GOPATH mode. return false - } else if search.InDir(modRoots[0], os.TempDir()) == "." { - _ = TODOWorkspaces("modRoots[0] is not right here") + } else if search.InDir(modRoot, os.TempDir()) == "." { // If you create /tmp/go.mod for experimenting, // then any tests that create work directories under /tmp // will find it and get modules when they're not expecting them. @@ -856,6 +864,7 @@ func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile panic("mainModulesCalled with module.Version with non empty Version field: " + fmt.Sprintf("%#v", m)) } } + modRootContainingCWD := findModuleRoot(base.Cwd()) mainModules := &MainModuleSet{ versions: ms[:len(ms):len(ms)], inGorootSrc: map[module.Version]bool{}, @@ -870,6 +879,10 @@ func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile mainModules.modFiles[m] = modFiles[i] mainModules.indices[m] = indices[i] + if mainModules.modRoot[m] == modRootContainingCWD { + mainModules.modContainingCWD = m + } + if rel := search.InDir(rootDirs[i], cfg.GOROOTsrc); rel != "" { mainModules.inGorootSrc[m] = true if m.Path == "std" { @@ -1108,7 +1121,7 @@ var altConfigs = []string{ ".git/config", } -func findModuleRoots(dir string) (roots []string) { +func findModuleRoot(dir string) (roots string) { if dir == "" { panic("dir not set") } @@ -1117,7 +1130,7 @@ func findModuleRoots(dir string) (roots []string) { // Look for enclosing go.mod. for { if fi, err := fsys.Stat(filepath.Join(dir, "go.mod")); err == nil && !fi.IsDir() { - return []string{dir} + return dir } d := filepath.Dir(dir) if d == dir { @@ -1125,7 +1138,7 @@ func findModuleRoots(dir string) (roots []string) { } dir = d } - return nil + return "" } func findWorkspaceFile(dir string) (root string) { diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 67d7ec65da2..7def3c2625a 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -1364,6 +1364,15 @@ func (ld *loader) resolveMissingImports(ctx context.Context) (modAddedBy map[mod var err error mod, err = queryImport(ctx, pkg.path, ld.requirements) if err != nil { + var ime *ImportMissingError + if errors.As(err, &ime) { + for curstack := pkg.stack; curstack != nil; curstack = curstack.stack { + if MainModules.Contains(curstack.mod.Path) { + ime.ImportingMainModule = curstack.mod + break + } + } + } // pkg.err was already non-nil, so we can reasonably attribute the error // for pkg to either the original error or the one returned by // queryImport. The existing error indicates only that we couldn't find diff --git a/src/cmd/go/testdata/script/work.txt b/src/cmd/go/testdata/script/work.txt index bcbabbacef8..9be09585796 100644 --- a/src/cmd/go/testdata/script/work.txt +++ b/src/cmd/go/testdata/script/work.txt @@ -2,7 +2,7 @@ go mod initwork ./a ./b cmp go.work go.work.want ! go run example.com/b -stderr 'a(\\|/)a.go:4:8: no required module provides package rsc.io/quote; to add it:\n\tgo get rsc.io/quote' +stderr 'a(\\|/)a.go:4:8: no required module provides package rsc.io/quote; to add it:\n\tcd '$WORK(\\|/)gopath(\\|/)src(\\|/)a'\n\tgo get rsc.io/quote' cd a go get rsc.io/quote go env GOMOD # go env GOMOD reports the module in a single module context From b3b53e1dad8681e3c21522513e4539813a5a3de7 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Sun, 4 Jul 2021 13:27:18 -0400 Subject: [PATCH 17/25] [dev.cmdgo] cmd/go: thread through modroots providing replacements modload.Replacement and modload.resolveReplacement now also return the modroot of the module providing a replacement so that we can correctly construct the path of a replaced module (because the path in the module.Version is relative to the modroot). For #45713 Change-Id: I8c69ccbcc1f40201071e35fcf93d6b5d0ed4cdf7 Reviewed-on: https://go-review.googlesource.com/c/go/+/334941 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modcmd/vendor.go | 3 +- src/cmd/go/internal/modget/get.go | 4 +- src/cmd/go/internal/modload/build.go | 14 ++--- src/cmd/go/internal/modload/import.go | 5 +- src/cmd/go/internal/modload/init.go | 14 +++-- src/cmd/go/internal/modload/load.go | 4 +- src/cmd/go/internal/modload/modfile.go | 72 +++++++++++++++----------- src/cmd/go/internal/modload/query.go | 10 ++-- src/cmd/go/internal/modload/vendor.go | 2 +- 9 files changed, 74 insertions(+), 54 deletions(-) diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 6273afbbe63..774fc3052fb 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -128,7 +128,8 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { } for _, m := range vendorMods { - line := moduleLine(m, modload.Replacement(m)) + replacement, _ := modload.Replacement(m) + line := moduleLine(m, replacement) io.WriteString(w, line) goVersion := "" diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 3c8b5a70900..6eae44f1a42 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -1615,7 +1615,7 @@ func (r *resolver) checkPackageProblems(ctx context.Context, pkgPatterns []strin i := i m := r.buildList[i] mActual := m - if mRepl := modload.Replacement(m); mRepl.Path != "" { + if mRepl, _ := modload.Replacement(m); mRepl.Path != "" { mActual = mRepl } old := module.Version{Path: m.Path, Version: r.initialVersion[m.Path]} @@ -1623,7 +1623,7 @@ func (r *resolver) checkPackageProblems(ctx context.Context, pkgPatterns []strin continue } oldActual := old - if oldRepl := modload.Replacement(old); oldRepl.Path != "" { + if oldRepl, _ := modload.Replacement(old); oldRepl.Path != "" { oldActual = oldRepl } if mActual == oldActual || mActual.Version == "" || !modfetch.HaveSum(oldActual) { diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index a5c5ad9b442..29735864790 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -241,7 +241,7 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li } // completeFromModCache fills in the extra fields in m using the module cache. - completeFromModCache := func(m *modinfo.ModulePublic) { + completeFromModCache := func(m *modinfo.ModulePublic, replacedFrom string) { checksumOk := func(suffix string) bool { return rs == nil || m.Version == "" || cfg.BuildMod == "mod" || modfetch.HaveSum(module.Version{Path: m.Path, Version: m.Version + suffix}) @@ -260,7 +260,7 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li if m.GoVersion == "" && checksumOk("/go.mod") { // Load the go.mod file to determine the Go version, since it hasn't // already been populated from rawGoVersion. - if summary, err := rawGoModSummary(mod); err == nil && summary.goVersion != "" { + if summary, err := rawGoModSummary(mod, replacedFrom); err == nil && summary.goVersion != "" { m.GoVersion = summary.goVersion } } @@ -290,11 +290,11 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li if rs == nil { // If this was an explicitly-versioned argument to 'go mod download' or // 'go list -m', report the actual requested version, not its replacement. - completeFromModCache(info) // Will set m.Error in vendor mode. + completeFromModCache(info, "") // Will set m.Error in vendor mode. return info } - r := Replacement(m) + r, replacedFrom := Replacement(m) if r.Path == "" { if cfg.BuildMod == "vendor" { // It's tempting to fill in the "Dir" field to point within the vendor @@ -303,7 +303,7 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li // interleave packages from different modules if one module path is a // prefix of the other. } else { - completeFromModCache(info) + completeFromModCache(info, "") } return info } @@ -328,7 +328,7 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li info.Replace.GoMod = filepath.Join(info.Replace.Dir, "go.mod") } if cfg.BuildMod != "vendor" { - completeFromModCache(info.Replace) + completeFromModCache(info.Replace, replacedFrom) info.Dir = info.Replace.Dir info.GoMod = info.Replace.GoMod info.Retracted = info.Replace.Retracted @@ -368,7 +368,7 @@ func PackageBuildInfo(path string, deps []string) string { mv = "(devel)" } fmt.Fprintf(&buf, "%s\t%s\t%s", token, m.Path, mv) - if r := Replacement(m); r.Path == "" { + if r, _ := Replacement(m); r.Path == "" { fmt.Fprintf(&buf, "\t%s\n", modfetch.Sum(m)) } else { fmt.Fprintf(&buf, "\n=>\t%s\t%s\t%s\n", r.Path, r.Version, modfetch.Sum(r)) diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 088d0c14ecb..5741299281a 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -479,11 +479,12 @@ func queryImport(ctx context.Context, path string, rs *Requirements) (module.Ver // The package path is not valid to fetch remotely, // so it can only exist in a replaced module, // and we know from the above loop that it is not. + replacement, _ := Replacement(mods[0]) return module.Version{}, &PackageNotInModuleError{ Mod: mods[0], Query: "latest", Pattern: path, - Replacement: Replacement(mods[0]), + Replacement: replacement, } } @@ -652,7 +653,7 @@ func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, i if modRoot := MainModules.ModRoot(mod); modRoot != "" { return modRoot, true, nil } - if r := Replacement(mod); r.Path != "" { + if r, _ := Replacement(mod); r.Path != "" { if r.Version == "" { dir = r.Path if !filepath.IsAbs(dir) { diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 18b07cb1257..de8adbafc44 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -1475,7 +1475,8 @@ func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { if v, ok := rs.rootSelected(prefix); ok && v != "none" { m := module.Version{Path: prefix, Version: v} - keep[resolveReplacement(m)] = true + r, _ := resolveReplacement(m) + keep[r] = true } } continue @@ -1486,7 +1487,8 @@ func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums for prefix := pkg.path; prefix != "."; prefix = path.Dir(prefix) { if v := mg.Selected(prefix); v != "none" { m := module.Version{Path: prefix, Version: v} - keep[resolveReplacement(m)] = true + r, _ := resolveReplacement(m) + keep[r] = true } } } @@ -1498,7 +1500,7 @@ func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums // Save sums for the root modules (or their replacements), but don't // incur the cost of loading the graph just to find and retain the sums. for _, m := range rs.rootModules { - r := resolveReplacement(m) + r, _ := resolveReplacement(m) keep[modkey(r)] = true if which == addBuildListZipSums { keep[r] = true @@ -1511,13 +1513,15 @@ func keepSums(ctx context.Context, ld *loader, rs *Requirements, which whichSums // The requirements from m's go.mod file are present in the module graph, // so they are relevant to the MVS result regardless of whether m was // actually selected. - keep[modkey(resolveReplacement(m))] = true + r, _ := resolveReplacement(m) + keep[modkey(r)] = true } }) if which == addBuildListZipSums { for _, m := range mg.BuildList() { - keep[resolveReplacement(m)] = true + r, _ := resolveReplacement(m) + keep[r] = true } } } diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 7def3c2625a..dd69e2afbfe 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -582,7 +582,7 @@ func pathInModuleCache(ctx context.Context, dir string, rs *Requirements) string tryMod := func(m module.Version) (string, bool) { var root string var err error - if repl := Replacement(m); repl.Path != "" && repl.Version == "" { + if repl, _ := Replacement(m); repl.Path != "" && repl.Version == "" { root = repl.Path if !filepath.IsAbs(root) { root = filepath.Join(ModRoot(), root) @@ -1800,7 +1800,7 @@ func (ld *loader) checkMultiplePaths() { firstPath := map[module.Version]string{} for _, mod := range mods { - src := resolveReplacement(mod) + src, _ := resolveReplacement(mod) if prev, ok := firstPath[src]; !ok { firstPath[src] = mod.Path } else if prev != mod.Path { diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 31e79a36460..bc5c83dffce 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -171,7 +171,7 @@ func CheckRetractions(ctx context.Context, m module.Version) (err error) { // Cannot be retracted. return nil } - if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + if repl, _ := Replacement(module.Version{Path: m.Path}); repl.Path != "" { // All versions of the module were replaced. // Don't load retractions, since we'd just load the replacement. return nil @@ -188,11 +188,11 @@ func CheckRetractions(ctx context.Context, m module.Version) (err error) { // We load the raw file here: the go.mod file may have a different module // path that we expect if the module or its repository was renamed. // We still want to apply retractions to other aliases of the module. - rm, err := queryLatestVersionIgnoringRetractions(ctx, m.Path) + rm, replacedFrom, err := queryLatestVersionIgnoringRetractions(ctx, m.Path) if err != nil { return err } - summary, err := rawGoModSummary(rm) + summary, err := rawGoModSummary(rm, replacedFrom) if err != nil { return err } @@ -290,17 +290,17 @@ func CheckDeprecation(ctx context.Context, m module.Version) (deprecation string // Don't look up deprecation. return "", nil } - if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" { + if repl, _ := Replacement(module.Version{Path: m.Path}); repl.Path != "" { // All versions of the module were replaced. // We'll look up deprecation separately for the replacement. return "", nil } - latest, err := queryLatestVersionIgnoringRetractions(ctx, m.Path) + latest, replacedFrom, err := queryLatestVersionIgnoringRetractions(ctx, m.Path) if err != nil { return "", err } - summary, err := rawGoModSummary(latest) + summary, err := rawGoModSummary(latest, replacedFrom) if err != nil { return "", err } @@ -317,10 +317,11 @@ func replacement(mod module.Version, index *modFileIndex) (fromVersion string, t return "", module.Version{}, false } -// Replacement returns the replacement for mod, if any, from go.mod. +// Replacement returns the replacement for mod, if any, and and the module root +// directory of the main module containing the replace directive. // If there is no replacement for mod, Replacement returns // a module.Version with Path == "". -func Replacement(mod module.Version) module.Version { +func Replacement(mod module.Version) (module.Version, string) { _ = TODOWorkspaces("support replaces in the go.work file") foundFrom, found, foundModRoot := "", module.Version{}, "" for _, v := range MainModules.Versions() { @@ -331,22 +332,24 @@ func Replacement(mod module.Version) module.Version { _ = TODOWorkspaces("once the go.work file supports replaces, recommend them as a way to override conflicts") base.Errorf("conflicting replacements found for %v in workspace modules defined by %v and %v", mod, modFilePath(foundModRoot), modFilePath(modRoot)) - return found + return found, foundModRoot } found, foundModRoot = r, modRoot } } } - return found + return found, foundModRoot } // resolveReplacement returns the module actually used to load the source code // for m: either m itself, or the replacement for m (iff m is replaced). -func resolveReplacement(m module.Version) module.Version { - if r := Replacement(m); r.Path != "" { - return r +// It also returns the modroot of the module providing the replacement if +// one was found. +func resolveReplacement(m module.Version) (module.Version, string) { + if r, replacedFrom := Replacement(m); r.Path != "" { + return r, replacedFrom } - return m + return m, "" } // indexModFile rebuilds the index of modFile. @@ -533,7 +536,7 @@ func goModSummary(m module.Version) (*modFileSummary, error) { return summary, nil } - actual := resolveReplacement(m) + actual, replacedFrom := resolveReplacement(m) if HasModRoot() && cfg.BuildMod == "readonly" && !inWorkspaceMode() && actual.Version != "" { key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"} if !modfetch.HaveSum(key) { @@ -541,7 +544,7 @@ func goModSummary(m module.Version) (*modFileSummary, error) { return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion}) } } - summary, err := rawGoModSummary(actual) + summary, err := rawGoModSummary(actual, replacedFrom) if err != nil { return nil, err } @@ -605,18 +608,23 @@ func goModSummary(m module.Version) (*modFileSummary, error) { // its dependencies. // // rawGoModSummary cannot be used on the Target module. -func rawGoModSummary(m module.Version) (*modFileSummary, error) { + +func rawGoModSummary(m module.Version, replacedFrom string) (*modFileSummary, error) { if m.Path == "" && MainModules.Contains(m.Path) { panic("internal error: rawGoModSummary called on the Target module") } + type key struct { + m module.Version + replacedFrom string + } type cached struct { summary *modFileSummary err error } - c := rawGoModSummaryCache.Do(m, func() interface{} { + c := rawGoModSummaryCache.Do(key{m, replacedFrom}, func() interface{} { summary := new(modFileSummary) - name, data, err := rawGoModData(m) + name, data, err := rawGoModData(m, replacedFrom) if err != nil { return cached{nil, err} } @@ -666,12 +674,15 @@ var rawGoModSummaryCache par.Cache // module.Version → rawGoModSummary result // // Unlike rawGoModSummary, rawGoModData does not cache its results in memory. // Use rawGoModSummary instead unless you specifically need these bytes. -func rawGoModData(m module.Version) (name string, data []byte, err error) { +func rawGoModData(m module.Version, replacedFrom string) (name string, data []byte, err error) { if m.Version == "" { // m is a replacement module with only a file path. dir := m.Path if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) + if replacedFrom == "" { + panic(fmt.Errorf("missing module root of main module providing replacement with relative path: %v", dir)) + } + dir = filepath.Join(replacedFrom, dir) } name = filepath.Join(dir, "go.mod") if gomodActual, ok := fsys.OverlayPath(name); ok { @@ -706,19 +717,20 @@ func rawGoModData(m module.Version) (name string, data []byte, err error) { // // If the queried latest version is replaced, // queryLatestVersionIgnoringRetractions returns the replacement. -func queryLatestVersionIgnoringRetractions(ctx context.Context, path string) (latest module.Version, err error) { +func queryLatestVersionIgnoringRetractions(ctx context.Context, path string) (latest module.Version, replacedFrom string, err error) { type entry struct { - latest module.Version - err error + latest module.Version + replacedFrom string // if latest is a replacement + err error } e := latestVersionIgnoringRetractionsCache.Do(path, func() interface{} { ctx, span := trace.StartSpan(ctx, "queryLatestVersionIgnoringRetractions "+path) defer span.Done() - if repl := Replacement(module.Version{Path: path}); repl.Path != "" { + if repl, replFrom := Replacement(module.Version{Path: path}); repl.Path != "" { // All versions of the module were replaced. // No need to query. - return &entry{latest: repl} + return &entry{latest: repl, replacedFrom: replFrom} } // Find the latest version of the module. @@ -730,12 +742,12 @@ func queryLatestVersionIgnoringRetractions(ctx context.Context, path string) (la return &entry{err: err} } latest := module.Version{Path: path, Version: rev.Version} - if repl := resolveReplacement(latest); repl.Path != "" { - latest = repl + if repl, replFrom := resolveReplacement(latest); repl.Path != "" { + latest, replacedFrom = repl, replFrom } - return &entry{latest: latest} + return &entry{latest: latest, replacedFrom: replacedFrom} }).(*entry) - return e.latest, e.err + return e.latest, e.replacedFrom, e.err } var latestVersionIgnoringRetractionsCache par.Cache // path → queryLatestVersionIgnoringRetractions result diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index a7031856ae2..6d6bfe774cd 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -513,9 +513,10 @@ func QueryPackages(ctx context.Context, pattern, query string, current func(stri pkgMods, modOnly, err := QueryPattern(ctx, pattern, query, current, allowed) if len(pkgMods) == 0 && err == nil { + replacement, _ := Replacement(modOnly.Mod) return nil, &PackageNotInModuleError{ Mod: modOnly.Mod, - Replacement: Replacement(modOnly.Mod), + Replacement: replacement, Query: query, Pattern: pattern, } @@ -670,9 +671,10 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin if err := firstError(m); err != nil { return r, err } + replacement, _ := Replacement(r.Mod) return r, &PackageNotInModuleError{ Mod: r.Mod, - Replacement: Replacement(r.Mod), + Replacement: replacement, Query: query, Pattern: pattern, } @@ -964,7 +966,7 @@ func moduleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { // we don't need to verify it in go.sum. This makes 'go list -m -u' faster // and simpler. func versionHasGoMod(_ context.Context, m module.Version) (bool, error) { - _, data, err := rawGoModData(m) + _, data, err := rawGoModData(m, "") if err != nil { return false, err } @@ -1093,7 +1095,7 @@ func (rr *replacementRepo) Stat(rev string) (*modfetch.RevInfo, error) { } } - if r := Replacement(module.Version{Path: path, Version: v}); r.Path == "" { + if r, _ := Replacement(module.Version{Path: path, Version: v}); r.Path == "" { return info, err } return rr.replacementStat(v) diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go index 6dc8b6cf820..b2da3783ea1 100644 --- a/src/cmd/go/internal/modload/vendor.go +++ b/src/cmd/go/internal/modload/vendor.go @@ -209,7 +209,7 @@ func checkVendorConsistency(index *modFileIndex, modFile *modfile.File) { } for _, mod := range vendorReplaced { - r := Replacement(mod) + r, _ := Replacement(mod) if r == (module.Version{}) { vendErrorf(mod, "is marked as replaced in vendor/modules.txt, but not replaced in go.mod") continue From 3799012990a324db968b8f377a5faf541cf20ac7 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 16 Jun 2021 18:46:23 -0400 Subject: [PATCH 18/25] [dev.cmdgo] cmd/go: add go mod editwork command go mod editwork behaves similarly to go mod edit: it has flags to change the go version, and add and remove directory and replace directives. For #45713 Change-Id: I1c795c122bfe461d6e87dd731692e0bf1bbe2bf7 Reviewed-on: https://go-review.googlesource.com/c/go/+/334938 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/alldocs.go | 72 ++++++ src/cmd/go/internal/modcmd/editwork.go | 282 +++++++++++++++++++++++ src/cmd/go/internal/modcmd/mod.go | 1 + src/cmd/go/internal/modload/init.go | 6 + src/cmd/go/testdata/script/work_edit.txt | 157 +++++++++++++ 5 files changed, 518 insertions(+) create mode 100644 src/cmd/go/internal/modcmd/editwork.go create mode 100644 src/cmd/go/testdata/script/work_edit.txt diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index af1ee396841..e0973acbf45 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -1032,6 +1032,7 @@ // // download download modules to local cache // edit edit go.mod from tools or scripts +// editwork edit go.work from tools or scripts // graph print module requirement graph // init initialize new module in current directory // initwork initialize workspace file @@ -1191,6 +1192,77 @@ // See https://golang.org/ref/mod#go-mod-edit for more about 'go mod edit'. // // +// Edit go.work from tools or scripts +// +// Usage: +// +// go mod editwork [editing flags] [go.work] +// +// Editwork provides a command-line interface for editing go.work, +// for use primarily by tools or scripts. It only reads go.work; +// it does not look up information about the modules involved. +// If no file is specified, editwork looks for a go.work file in the current +// directory and its parent directories +// +// The editing flags specify a sequence of editing operations. +// +// The -fmt flag reformats the go.work file without making other changes. +// This reformatting is also implied by any other modifications that use or +// rewrite the go.mod file. The only time this flag is needed is if no other +// flags are specified, as in 'go mod editwork -fmt'. +// +// The -directory=path and -dropdirectory=path flags +// add and drop a directory from the go.work files set of module directories. +// +// The -replace=old[@v]=new[@v] flag adds a replacement of the given +// module path and version pair. If the @v in old@v is omitted, a +// replacement without a version on the left side is added, which applies +// to all versions of the old module path. If the @v in new@v is omitted, +// the new path should be a local module root directory, not a module +// path. Note that -replace overrides any redundant replacements for old[@v], +// so omitting @v will drop existing replacements for specific versions. +// +// The -dropreplace=old[@v] flag drops a replacement of the given +// module path and version pair. If the @v is omitted, a replacement without +// a version on the left side is dropped. +// +// The -directory, -dropdirectory, -replace, and -dropreplace, +// editing flags may be repeated, and the changes are applied in the order given. +// +// The -go=version flag sets the expected Go language version. +// +// The -print flag prints the final go.work in its text format instead of +// writing it back to go.mod. +// +// The -json flag prints the final go.work file in JSON format instead of +// writing it back to go.mod. The JSON output corresponds to these Go types: +// +// type Module struct { +// Path string +// Version string +// } +// +// type GoWork struct { +// Go string +// Directory []Directory +// Replace []Replace +// } +// +// type Directory struct { +// Path string +// ModulePath string +// } +// +// type Replace struct { +// Old Module +// New Module +// } +// +// See the workspaces design proposal at +// https://go.googlesource.com/proposal/+/master/design/45713-workspace.md for +// more information. +// +// // Print module requirement graph // // Usage: diff --git a/src/cmd/go/internal/modcmd/editwork.go b/src/cmd/go/internal/modcmd/editwork.go new file mode 100644 index 00000000000..f05d9245e7d --- /dev/null +++ b/src/cmd/go/internal/modcmd/editwork.go @@ -0,0 +1,282 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// go mod editwork + +package modcmd + +import ( + "bytes" + "cmd/go/internal/base" + "cmd/go/internal/lockedfile" + "cmd/go/internal/modload" + "context" + "encoding/json" + "errors" + "os" + "strings" + + "golang.org/x/mod/modfile" +) + +var cmdEditwork = &base.Command{ + UsageLine: "go mod editwork [editing flags] [go.work]", + Short: "edit go.work from tools or scripts", + Long: `Editwork provides a command-line interface for editing go.work, +for use primarily by tools or scripts. It only reads go.work; +it does not look up information about the modules involved. +If no file is specified, editwork looks for a go.work file in the current +directory and its parent directories + +The editing flags specify a sequence of editing operations. + +The -fmt flag reformats the go.work file without making other changes. +This reformatting is also implied by any other modifications that use or +rewrite the go.mod file. The only time this flag is needed is if no other +flags are specified, as in 'go mod editwork -fmt'. + +The -directory=path and -dropdirectory=path flags +add and drop a directory from the go.work files set of module directories. + +The -replace=old[@v]=new[@v] flag adds a replacement of the given +module path and version pair. If the @v in old@v is omitted, a +replacement without a version on the left side is added, which applies +to all versions of the old module path. If the @v in new@v is omitted, +the new path should be a local module root directory, not a module +path. Note that -replace overrides any redundant replacements for old[@v], +so omitting @v will drop existing replacements for specific versions. + +The -dropreplace=old[@v] flag drops a replacement of the given +module path and version pair. If the @v is omitted, a replacement without +a version on the left side is dropped. + +The -directory, -dropdirectory, -replace, and -dropreplace, +editing flags may be repeated, and the changes are applied in the order given. + +The -go=version flag sets the expected Go language version. + +The -print flag prints the final go.work in its text format instead of +writing it back to go.mod. + +The -json flag prints the final go.work file in JSON format instead of +writing it back to go.mod. The JSON output corresponds to these Go types: + + type Module struct { + Path string + Version string + } + + type GoWork struct { + Go string + Directory []Directory + Replace []Replace + } + + type Directory struct { + Path string + ModulePath string + } + + type Replace struct { + Old Module + New Module + } + +See the workspaces design proposal at +https://go.googlesource.com/proposal/+/master/design/45713-workspace.md for +more information. +`, +} + +var ( + editworkFmt = cmdEditwork.Flag.Bool("fmt", false, "") + editworkGo = cmdEditwork.Flag.String("go", "", "") + editworkJSON = cmdEditwork.Flag.Bool("json", false, "") + editworkPrint = cmdEditwork.Flag.Bool("print", false, "") + workedits []func(file *modfile.WorkFile) // edits specified in flags +) + +func init() { + cmdEditwork.Run = runEditwork // break init cycle + + cmdEditwork.Flag.Var(flagFunc(flagEditworkDirectory), "directory", "") + cmdEditwork.Flag.Var(flagFunc(flagEditworkDropDirectory), "dropdirectory", "") + cmdEditwork.Flag.Var(flagFunc(flagEditworkReplace), "replace", "") + cmdEditwork.Flag.Var(flagFunc(flagEditworkDropReplace), "dropreplace", "") + + base.AddWorkfileFlag(&cmdEditwork.Flag) +} + +func runEditwork(ctx context.Context, cmd *base.Command, args []string) { + anyFlags := + *editworkGo != "" || + *editworkJSON || + *editworkPrint || + *editworkFmt || + len(workedits) > 0 + + if !anyFlags { + base.Fatalf("go mod edit: no flags specified (see 'go help mod edit').") + } + + if *editworkJSON && *editworkPrint { + base.Fatalf("go mod edit: cannot use both -json and -print") + } + + if len(args) > 1 { + base.Fatalf("go mod edit: too many arguments") + } + var gowork string + if len(args) == 1 { + gowork = args[0] + } else { + modload.InitWorkfile() + gowork = modload.WorkFilePath() + } + + if *editworkGo != "" { + if !modfile.GoVersionRE.MatchString(*editworkGo) { + base.Fatalf(`go mod: invalid -go option; expecting something like "-go %s"`, modload.LatestGoVersion()) + } + } + + data, err := lockedfile.Read(gowork) + if err != nil { + base.Fatalf("go: %v", err) + } + + workFile, err := modfile.ParseWork(gowork, data, nil) + if err != nil { + base.Fatalf("go: errors parsing %s:\n%s", base.ShortPath(gowork), err) + } + + if *editworkGo != "" { + if err := workFile.AddGoStmt(*editworkGo); err != nil { + base.Fatalf("go: internal error: %v", err) + } + } + + if len(workedits) > 0 { + for _, edit := range workedits { + edit(workFile) + } + } + workFile.SortBlocks() + workFile.Cleanup() // clean file after edits + + if *editworkJSON { + editworkPrintJSON(workFile) + return + } + + out := modfile.Format(workFile.Syntax) + + if *editworkPrint { + os.Stdout.Write(out) + return + } + + err = lockedfile.Transform(gowork, func(lockedData []byte) ([]byte, error) { + if !bytes.Equal(lockedData, data) { + return nil, errors.New("go.work changed during editing; not overwriting") + } + return out, nil + }) + if err != nil { + base.Fatalf("go: %v", err) + } +} + +// flagEditworkDirectory implements the -directory flag. +func flagEditworkDirectory(arg string) { + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.AddDirectory(arg, ""); err != nil { + base.Fatalf("go mod: -directory=%s: %v", arg, err) + } + }) +} + +// flagEditworkDropDirectory implements the -dropdirectory flag. +func flagEditworkDropDirectory(arg string) { + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.DropDirectory(arg); err != nil { + base.Fatalf("go mod: -dropdirectory=%s: %v", arg, err) + } + }) +} + +// flagReplace implements the -replace flag. +func flagEditworkReplace(arg string) { + var i int + if i = strings.Index(arg, "="); i < 0 { + base.Fatalf("go mod: -replace=%s: need old[@v]=new[@w] (missing =)", arg) + } + old, new := strings.TrimSpace(arg[:i]), strings.TrimSpace(arg[i+1:]) + if strings.HasPrefix(new, ">") { + base.Fatalf("go mod: -replace=%s: separator between old and new is =, not =>", arg) + } + oldPath, oldVersion, err := parsePathVersionOptional("old", old, false) + if err != nil { + base.Fatalf("go mod: -replace=%s: %v", arg, err) + } + newPath, newVersion, err := parsePathVersionOptional("new", new, true) + if err != nil { + base.Fatalf("go mod: -replace=%s: %v", arg, err) + } + if newPath == new && !modfile.IsDirectoryPath(new) { + base.Fatalf("go mod: -replace=%s: unversioned new path must be local directory", arg) + } + + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.AddReplace(oldPath, oldVersion, newPath, newVersion); err != nil { + base.Fatalf("go mod: -replace=%s: %v", arg, err) + } + }) +} + +// flagDropReplace implements the -dropreplace flag. +func flagEditworkDropReplace(arg string) { + path, version, err := parsePathVersionOptional("old", arg, true) + if err != nil { + base.Fatalf("go mod: -dropreplace=%s: %v", arg, err) + } + workedits = append(workedits, func(f *modfile.WorkFile) { + if err := f.DropReplace(path, version); err != nil { + base.Fatalf("go mod: -dropreplace=%s: %v", arg, err) + } + }) +} + +// editPrintJSON prints the -json output. +func editworkPrintJSON(workFile *modfile.WorkFile) { + var f workfileJSON + if workFile.Go != nil { + f.Go = workFile.Go.Version + } + for _, d := range workFile.Directory { + f.Directory = append(f.Directory, directoryJSON{DiskPath: d.Path, ModPath: d.ModulePath}) + } + + for _, r := range workFile.Replace { + f.Replace = append(f.Replace, replaceJSON{r.Old, r.New}) + } + data, err := json.MarshalIndent(&f, "", "\t") + if err != nil { + base.Fatalf("go: internal error: %v", err) + } + data = append(data, '\n') + os.Stdout.Write(data) +} + +// workfileJSON is the -json output data structure. +type workfileJSON struct { + Go string `json:",omitempty"` + Directory []directoryJSON + Replace []replaceJSON +} + +type directoryJSON struct { + DiskPath string + ModPath string `json:",omitempty"` +} diff --git a/src/cmd/go/internal/modcmd/mod.go b/src/cmd/go/internal/modcmd/mod.go index 3586b44c1ad..29aad583240 100644 --- a/src/cmd/go/internal/modcmd/mod.go +++ b/src/cmd/go/internal/modcmd/mod.go @@ -23,6 +23,7 @@ See 'go help modules' for an overview of module functionality. Commands: []*base.Command{ cmdDownload, cmdEdit, + cmdEditwork, cmdGraph, cmdInit, cmdInitwork, diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index de8adbafc44..5dd946215bf 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -249,6 +249,12 @@ func InitWorkfile() { } } +// WorkFilePath returns the path of the go.work file, or "" if not in +// workspace mode. WorkFilePath must be called after InitWorkfile. +func WorkFilePath() string { + return workFilePath +} + // Init determines whether module mode is enabled, locates the root of the // current module (if any), sets environment variables for Git subprocesses, and // configures the cfg, codehost, load, modfetch, and search packages for use diff --git a/src/cmd/go/testdata/script/work_edit.txt b/src/cmd/go/testdata/script/work_edit.txt new file mode 100644 index 00000000000..0717086ee7c --- /dev/null +++ b/src/cmd/go/testdata/script/work_edit.txt @@ -0,0 +1,157 @@ +# Test editing go.work files. + +go mod initwork m +cmp go.work go.work.want_initial + +go mod editwork -directory n +cmp go.work go.work.want_directory_n + +go mod editwork -go 1.18 +cmp go.work go.work.want_go_118 + +go mod editwork -dropdirectory m +cmp go.work go.work.want_dropdirectory_m + +go mod editwork -replace=x.1@v1.3.0=y.1@v1.4.0 -replace='x.1@v1.4.0 = ../z' +cmp go.work go.work.want_add_replaces + +go mod editwork -directory n -directory ../a -directory /b -directory c -directory c +cmp go.work go.work.want_multidirectory + +go mod editwork -dropdirectory /b -dropdirectory n +cmp go.work go.work.want_multidropdirectory + +go mod editwork -dropreplace='x.1@v1.4.0' +cmp go.work go.work.want_dropreplace + +go mod editwork -print -go 1.19 -directory b -dropdirectory c -replace 'x.1@v1.4.0 = ../z' -dropreplace x.1 -dropreplace x.1@v1.3.0 +cmp stdout go.work.want_print + +go mod editwork -json -go 1.19 -directory b -dropdirectory c -replace 'x.1@v1.4.0 = ../z' -dropreplace x.1 -dropreplace x.1@v1.3.0 +cmp stdout go.work.want_json + +go mod editwork -print -fmt -workfile unformatted +cmp stdout formatted + +-- go.work.want_initial -- +go 1.17 + +directory m +-- go.work.want_directory_n -- +go 1.17 + +directory ( + m + n +) +-- go.work.want_go_118 -- +go 1.18 + +directory ( + m + n +) +-- go.work.want_dropdirectory_m -- +go 1.18 + +directory n +-- go.work.want_add_replaces -- +go 1.18 + +directory n + +replace ( + x.1 v1.3.0 => y.1 v1.4.0 + x.1 v1.4.0 => ../z +) +-- go.work.want_multidirectory -- +go 1.18 + +directory ( + ../a + /b + c + n +) + +replace ( + x.1 v1.3.0 => y.1 v1.4.0 + x.1 v1.4.0 => ../z +) +-- go.work.want_multidropdirectory -- +go 1.18 + +directory ( + ../a + c +) + +replace ( + x.1 v1.3.0 => y.1 v1.4.0 + x.1 v1.4.0 => ../z +) +-- go.work.want_dropreplace -- +go 1.18 + +directory ( + ../a + c +) + +replace x.1 v1.3.0 => y.1 v1.4.0 +-- go.work.want_print -- +go 1.19 + +directory ( + ../a + b +) + +replace x.1 v1.4.0 => ../z +-- go.work.want_json -- +{ + "Go": "1.19", + "Directory": [ + { + "DiskPath": "../a" + }, + { + "DiskPath": "b" + } + ], + "Replace": [ + { + "Old": { + "Path": "x.1", + "Version": "v1.4.0" + }, + "New": { + "Path": "../z" + } + } + ] +} +-- unformatted -- +go 1.17 + directory ( + a + b + c + ) + replace ( + x.1 v1.3.0 => y.1 v1.4.0 + x.1 v1.4.0 => ../z + ) +-- formatted -- +go 1.17 + +directory ( + a + b + c +) + +replace ( + x.1 v1.3.0 => y.1 v1.4.0 + x.1 v1.4.0 => ../z +) \ No newline at end of file From fc8e0cbbbaa5ae7c1e5b7b070ad80f41095ef18b Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 26 Jul 2021 10:39:38 -0700 Subject: [PATCH 19/25] [dev.cmdgo] cmd: update x/tools and remove copy of txtar golang.org/x/tools/txtar is the main location for this package. We don't need our own copy. Also, update cmd/vet tests for compatibility with CL 301949. For golang/go#47193 Change-Id: I480eb591f57a0d05b433a657653e2021e39354eb Reviewed-on: https://go-review.googlesource.com/c/go/+/337352 Trust: Jay Conrod Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- src/cmd/go.mod | 2 +- src/cmd/go.sum | 4 +- src/cmd/go/internal/fsys/fsys_test.go | 3 +- src/cmd/go/internal/txtar/archive_test.go | 67 ----------- src/cmd/go/proxy_test.go | 2 +- src/cmd/go/script_test.go | 3 +- src/cmd/go/testdata/addmod.go | 2 +- src/cmd/go/testdata/savedir.go | 2 +- .../tools/go/analysis/passes/printf/printf.go | 28 +++-- .../x/tools/go/ast/astutil/rewrite.go | 8 +- .../x/tools/internal/typeparams/doc.go | 11 ++ .../tools/internal/typeparams/notypeparams.go | 90 +++++++++++++++ .../x/tools/internal/typeparams/typeparams.go | 105 ++++++++++++++++++ .../golang.org/x/tools}/txtar/archive.go | 6 +- src/cmd/vendor/modules.txt | 4 +- src/cmd/vet/testdata/print/print.go | 8 +- 16 files changed, 249 insertions(+), 96 deletions(-) delete mode 100644 src/cmd/go/internal/txtar/archive_test.go create mode 100644 src/cmd/vendor/golang.org/x/tools/internal/typeparams/doc.go create mode 100644 src/cmd/vendor/golang.org/x/tools/internal/typeparams/notypeparams.go create mode 100644 src/cmd/vendor/golang.org/x/tools/internal/typeparams/typeparams.go rename src/cmd/{go/internal => vendor/golang.org/x/tools}/txtar/archive.go (96%) diff --git a/src/cmd/go.mod b/src/cmd/go.mod index 21d7d8b75a2..b0a3f480458 100644 --- a/src/cmd/go.mod +++ b/src/cmd/go.mod @@ -10,6 +10,6 @@ require ( golang.org/x/mod v0.4.3-0.20210723200715-e41a6a4f3b61 golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744 // indirect golang.org/x/term v0.0.0-20210503060354-a79de5458b56 - golang.org/x/tools v0.1.2-0.20210519160823-49064d2332f9 + golang.org/x/tools v0.1.6-0.20210726171848-ebce39e5e3d6 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) diff --git a/src/cmd/go.sum b/src/cmd/go.sum index 529b152b771..bf237d40bcf 100644 --- a/src/cmd/go.sum +++ b/src/cmd/go.sum @@ -36,8 +36,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.2-0.20210519160823-49064d2332f9 h1:2XlR/j4I4xz5GQZI7zBjqTfezYyRIE2jD5IMousB2rg= -golang.org/x/tools v0.1.2-0.20210519160823-49064d2332f9/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726171848-ebce39e5e3d6 h1:er++nfKy5Irv8JPDfJ68QzoVKQ6MBF7cf5xC15O4Zy0= +golang.org/x/tools v0.1.6-0.20210726171848-ebce39e5e3d6/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= diff --git a/src/cmd/go/internal/fsys/fsys_test.go b/src/cmd/go/internal/fsys/fsys_test.go index 7f175c70311..c080c14987c 100644 --- a/src/cmd/go/internal/fsys/fsys_test.go +++ b/src/cmd/go/internal/fsys/fsys_test.go @@ -1,7 +1,6 @@ package fsys import ( - "cmd/go/internal/txtar" "encoding/json" "errors" "fmt" @@ -12,6 +11,8 @@ import ( "path/filepath" "reflect" "testing" + + "golang.org/x/tools/txtar" ) // initOverlay resets the overlay state to reflect the config. diff --git a/src/cmd/go/internal/txtar/archive_test.go b/src/cmd/go/internal/txtar/archive_test.go deleted file mode 100644 index 3f734f67625..00000000000 --- a/src/cmd/go/internal/txtar/archive_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package txtar - -import ( - "bytes" - "fmt" - "reflect" - "testing" -) - -var tests = []struct { - name string - text string - parsed *Archive -}{ - { - name: "basic", - text: `comment1 -comment2 --- file1 -- -File 1 text. --- foo --- -More file 1 text. --- file 2 -- -File 2 text. --- empty -- --- noNL -- -hello world`, - parsed: &Archive{ - Comment: []byte("comment1\ncomment2\n"), - Files: []File{ - {"file1", []byte("File 1 text.\n-- foo ---\nMore file 1 text.\n")}, - {"file 2", []byte("File 2 text.\n")}, - {"empty", []byte{}}, - {"noNL", []byte("hello world\n")}, - }, - }, - }, -} - -func Test(t *testing.T) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - a := Parse([]byte(tt.text)) - if !reflect.DeepEqual(a, tt.parsed) { - t.Fatalf("Parse: wrong output:\nhave:\n%s\nwant:\n%s", shortArchive(a), shortArchive(tt.parsed)) - } - text := Format(a) - a = Parse(text) - if !reflect.DeepEqual(a, tt.parsed) { - t.Fatalf("Parse after Format: wrong output:\nhave:\n%s\nwant:\n%s", shortArchive(a), shortArchive(tt.parsed)) - } - }) - } -} - -func shortArchive(a *Archive) string { - var buf bytes.Buffer - fmt.Fprintf(&buf, "comment: %q\n", a.Comment) - for _, f := range a.Files { - fmt.Fprintf(&buf, "file %q: %q\n", f.Name, f.Data) - } - return buf.String() -} diff --git a/src/cmd/go/proxy_test.go b/src/cmd/go/proxy_test.go index 74bfecc08db..a387fe67dbf 100644 --- a/src/cmd/go/proxy_test.go +++ b/src/cmd/go/proxy_test.go @@ -25,12 +25,12 @@ import ( "cmd/go/internal/modfetch/codehost" "cmd/go/internal/par" - "cmd/go/internal/txtar" "golang.org/x/mod/module" "golang.org/x/mod/semver" "golang.org/x/mod/sumdb" "golang.org/x/mod/sumdb/dirhash" + "golang.org/x/tools/txtar" ) var ( diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index 8a7c77a46fa..3c5855bd6f2 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -31,9 +31,10 @@ import ( "cmd/go/internal/imports" "cmd/go/internal/par" "cmd/go/internal/robustio" - "cmd/go/internal/txtar" "cmd/go/internal/work" "cmd/internal/sys" + + "golang.org/x/tools/txtar" ) var testSum = flag.String("testsum", "", `may be tidy, listm, or listall. If set, TestScript generates a go.sum file at the beginning of each test and updates test files if they pass.`) diff --git a/src/cmd/go/testdata/addmod.go b/src/cmd/go/testdata/addmod.go index 03869e68def..a1ace4ce590 100644 --- a/src/cmd/go/testdata/addmod.go +++ b/src/cmd/go/testdata/addmod.go @@ -29,7 +29,7 @@ import ( "path/filepath" "strings" - "cmd/go/internal/txtar" + "golang.org/x/tools/txtar" ) func usage() { diff --git a/src/cmd/go/testdata/savedir.go b/src/cmd/go/testdata/savedir.go index d469c31a919..6a8a232702f 100644 --- a/src/cmd/go/testdata/savedir.go +++ b/src/cmd/go/testdata/savedir.go @@ -24,7 +24,7 @@ import ( "strings" "unicode/utf8" - "../internal/txtar" + "golang.org/x/tools/txtar" ) func usage() { diff --git a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/printf.go b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/printf.go index 822820f06e9..6589478af0f 100644 --- a/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/printf.go +++ b/src/cmd/vendor/golang.org/x/tools/go/analysis/passes/printf/printf.go @@ -555,7 +555,7 @@ func checkPrintf(pass *analysis.Pass, kind Kind, call *ast.CallExpr, fn *types.F format, idx := formatString(pass, call) if idx < 0 { if false { - pass.Reportf(call.Lparen, "can't check non-constant format in call to %s", fn.Name()) + pass.Reportf(call.Lparen, "can't check non-constant format in call to %s", fn.FullName()) } return } @@ -563,7 +563,7 @@ func checkPrintf(pass *analysis.Pass, kind Kind, call *ast.CallExpr, fn *types.F firstArg := idx + 1 // Arguments are immediately after format string. if !strings.Contains(format, "%") { if len(call.Args) > firstArg { - pass.Reportf(call.Lparen, "%s call has arguments but no formatting directives", fn.Name()) + pass.Reportf(call.Lparen, "%s call has arguments but no formatting directives", fn.FullName()) } return } @@ -577,7 +577,7 @@ func checkPrintf(pass *analysis.Pass, kind Kind, call *ast.CallExpr, fn *types.F if format[i] != '%' { continue } - state := parsePrintfVerb(pass, call, fn.Name(), format[i:], firstArg, argNum) + state := parsePrintfVerb(pass, call, fn.FullName(), format[i:], firstArg, argNum) if state == nil { return } @@ -589,8 +589,12 @@ func checkPrintf(pass *analysis.Pass, kind Kind, call *ast.CallExpr, fn *types.F anyIndex = true } if state.verb == 'w' { - if kind != KindErrorf { - pass.Reportf(call.Pos(), "%s call has error-wrapping directive %%w, which is only supported by Errorf", state.name) + switch kind { + case KindNone, KindPrint: + pass.Reportf(call.Pos(), "%s does not support error-wrapping directive %%w", state.name) + return + case KindPrintf: + pass.Reportf(call.Pos(), "%s call has error-wrapping directive %%w, which is only supported for functions backed by fmt.Errorf", state.name) return } if anyW { @@ -621,7 +625,7 @@ func checkPrintf(pass *analysis.Pass, kind Kind, call *ast.CallExpr, fn *types.F if maxArgNum != len(call.Args) { expect := maxArgNum - firstArg numArgs := len(call.Args) - firstArg - pass.ReportRangef(call, "%s call needs %v but has %v", fn.Name(), count(expect, "arg"), count(numArgs, "arg")) + pass.ReportRangef(call, "%s call needs %v but has %v", fn.FullName(), count(expect, "arg"), count(numArgs, "arg")) } } @@ -949,7 +953,7 @@ func recursiveStringer(pass *analysis.Pass, e ast.Expr) (string, bool) { } if id, ok := e.(*ast.Ident); ok { if pass.TypesInfo.Uses[id] == sig.Recv() { - return method.Name(), true + return method.FullName(), true } } return "", false @@ -1044,7 +1048,7 @@ func checkPrint(pass *analysis.Pass, call *ast.CallExpr, fn *types.Func) { if sel, ok := call.Args[0].(*ast.SelectorExpr); ok { if x, ok := sel.X.(*ast.Ident); ok { if x.Name == "os" && strings.HasPrefix(sel.Sel.Name, "Std") { - pass.ReportRangef(call, "%s does not take io.Writer but has first arg %s", fn.Name(), analysisutil.Format(pass.Fset, call.Args[0])) + pass.ReportRangef(call, "%s does not take io.Writer but has first arg %s", fn.FullName(), analysisutil.Format(pass.Fset, call.Args[0])) } } } @@ -1058,7 +1062,7 @@ func checkPrint(pass *analysis.Pass, call *ast.CallExpr, fn *types.Func) { if strings.Contains(s, "%") { m := printFormatRE.FindStringSubmatch(s) if m != nil { - pass.ReportRangef(call, "%s call has possible formatting directive %s", fn.Name(), m[0]) + pass.ReportRangef(call, "%s call has possible formatting directive %s", fn.FullName(), m[0]) } } } @@ -1068,16 +1072,16 @@ func checkPrint(pass *analysis.Pass, call *ast.CallExpr, fn *types.Func) { if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING { str, _ := strconv.Unquote(lit.Value) if strings.HasSuffix(str, "\n") { - pass.ReportRangef(call, "%s arg list ends with redundant newline", fn.Name()) + pass.ReportRangef(call, "%s arg list ends with redundant newline", fn.FullName()) } } } for _, arg := range args { if isFunctionValue(pass, arg) { - pass.ReportRangef(call, "%s arg %s is a func value, not called", fn.Name(), analysisutil.Format(pass.Fset, arg)) + pass.ReportRangef(call, "%s arg %s is a func value, not called", fn.FullName(), analysisutil.Format(pass.Fset, arg)) } if methodName, ok := recursiveStringer(pass, arg); ok { - pass.ReportRangef(call, "%s arg %s causes recursive call to %s method", fn.Name(), analysisutil.Format(pass.Fset, arg), methodName) + pass.ReportRangef(call, "%s arg %s causes recursive call to %s method", fn.FullName(), analysisutil.Format(pass.Fset, arg), methodName) } } } diff --git a/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/rewrite.go b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/rewrite.go index cf72ea990bd..b949fc84079 100644 --- a/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/rewrite.go +++ b/src/cmd/vendor/golang.org/x/tools/go/ast/astutil/rewrite.go @@ -9,6 +9,8 @@ import ( "go/ast" "reflect" "sort" + + "golang.org/x/tools/internal/typeparams" ) // An ApplyFunc is invoked by Apply for each node n, even if n is nil, @@ -437,7 +439,11 @@ func (a *application) apply(parent ast.Node, name string, iter *iterator, n ast. } default: - panic(fmt.Sprintf("Apply: unexpected node type %T", n)) + if typeparams.IsListExpr(n) { + a.applyList(n, "ElemList") + } else { + panic(fmt.Sprintf("Apply: unexpected node type %T", n)) + } } if a.post != nil && !a.post(&a.cursor) { diff --git a/src/cmd/vendor/golang.org/x/tools/internal/typeparams/doc.go b/src/cmd/vendor/golang.org/x/tools/internal/typeparams/doc.go new file mode 100644 index 00000000000..5583947e21f --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/internal/typeparams/doc.go @@ -0,0 +1,11 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package typeparams provides functions to work indirectly with type parameter +// data stored in go/ast and go/types objects, while these API are guarded by a +// build constraint. +// +// This package exists to make it easier for tools to work with generic code, +// while also compiling against older Go versions. +package typeparams diff --git a/src/cmd/vendor/golang.org/x/tools/internal/typeparams/notypeparams.go b/src/cmd/vendor/golang.org/x/tools/internal/typeparams/notypeparams.go new file mode 100644 index 00000000000..3a0abc7c18e --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/internal/typeparams/notypeparams.go @@ -0,0 +1,90 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !typeparams || !go1.17 +// +build !typeparams !go1.17 + +package typeparams + +import ( + "go/ast" + "go/types" +) + +// NOTE: doc comments must be kept in sync with typeparams.go. + +// Enabled reports whether type parameters are enabled in the current build +// environment. +const Enabled = false + +// UnpackIndex extracts all index expressions from e. For non-generic code this +// is always one expression: e.Index, but may be more than one expression for +// generic type instantiation. +func UnpackIndex(e *ast.IndexExpr) []ast.Expr { + return []ast.Expr{e.Index} +} + +// IsListExpr reports whether n is an *ast.ListExpr, which is a new node type +// introduced to hold type arguments for generic type instantiation. +func IsListExpr(n ast.Node) bool { + return false +} + +// ForTypeDecl extracts the (possibly nil) type parameter node list from n. +func ForTypeDecl(*ast.TypeSpec) *ast.FieldList { + return nil +} + +// ForFuncDecl extracts the (possibly nil) type parameter node list from n. +func ForFuncDecl(*ast.FuncDecl) *ast.FieldList { + return nil +} + +// ForSignature extracts the (possibly empty) type parameter object list from +// sig. +func ForSignature(*types.Signature) []*types.TypeName { + return nil +} + +// HasTypeSet reports if iface has a type set. +func HasTypeSet(*types.Interface) bool { + return false +} + +// IsComparable reports if iface is the comparable interface. +func IsComparable(*types.Interface) bool { + return false +} + +// IsConstraint reports whether iface may only be used as a type parameter +// constraint (i.e. has a type set or is the comparable interface). +func IsConstraint(*types.Interface) bool { + return false +} + +// ForNamed extracts the (possibly empty) type parameter object list from +// named. +func ForNamed(*types.Named) []*types.TypeName { + return nil +} + +// NamedTArgs extracts the (possibly empty) type argument list from named. +func NamedTArgs(*types.Named) []types.Type { + return nil +} + +// InitInferred initializes info to record inferred type information. +func InitInferred(*types.Info) { +} + +// GetInferred extracts inferred type information from info for e. +// +// The expression e may have an inferred type if it is an *ast.IndexExpr +// representing partial instantiation of a generic function type for which type +// arguments have been inferred using constraint type inference, or if it is an +// *ast.CallExpr for which type type arguments have be inferred using both +// constraint type inference and function argument inference. +func GetInferred(*types.Info, ast.Expr) ([]types.Type, *types.Signature) { + return nil, nil +} diff --git a/src/cmd/vendor/golang.org/x/tools/internal/typeparams/typeparams.go b/src/cmd/vendor/golang.org/x/tools/internal/typeparams/typeparams.go new file mode 100644 index 00000000000..6b7958af060 --- /dev/null +++ b/src/cmd/vendor/golang.org/x/tools/internal/typeparams/typeparams.go @@ -0,0 +1,105 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build typeparams && go1.17 +// +build typeparams,go1.17 + +package typeparams + +import ( + "go/ast" + "go/types" +) + +// NOTE: doc comments must be kept in sync with notypeparams.go. + +// Enabled reports whether type parameters are enabled in the current build +// environment. +const Enabled = true + +// UnpackIndex extracts all index expressions from e. For non-generic code this +// is always one expression: e.Index, but may be more than one expression for +// generic type instantiation. +func UnpackIndex(e *ast.IndexExpr) []ast.Expr { + if x, _ := e.Index.(*ast.ListExpr); x != nil { + return x.ElemList + } + if e.Index != nil { + return []ast.Expr{e.Index} + } + return nil +} + +// IsListExpr reports whether n is an *ast.ListExpr, which is a new node type +// introduced to hold type arguments for generic type instantiation. +func IsListExpr(n ast.Node) bool { + _, ok := n.(*ast.ListExpr) + return ok +} + +// ForTypeDecl extracts the (possibly nil) type parameter node list from n. +func ForTypeDecl(n *ast.TypeSpec) *ast.FieldList { + return n.TParams +} + +// ForFuncDecl extracts the (possibly nil) type parameter node list from n. +func ForFuncDecl(n *ast.FuncDecl) *ast.FieldList { + if n.Type != nil { + return n.Type.TParams + } + return nil +} + +// ForSignature extracts the (possibly empty) type parameter object list from +// sig. +func ForSignature(sig *types.Signature) []*types.TypeName { + return sig.TParams() +} + +// HasTypeSet reports if iface has a type set. +func HasTypeSet(iface *types.Interface) bool { + return iface.HasTypeList() +} + +// IsComparable reports if iface is the comparable interface. +func IsComparable(iface *types.Interface) bool { + return iface.IsComparable() +} + +// IsConstraint reports whether iface may only be used as a type parameter +// constraint (i.e. has a type set or is the comparable interface). +func IsConstraint(iface *types.Interface) bool { + return iface.IsConstraint() +} + +// ForNamed extracts the (possibly empty) type parameter object list from +// named. +func ForNamed(named *types.Named) []*types.TypeName { + return named.TParams() +} + +// NamedTArgs extracts the (possibly empty) type argument list from named. +func NamedTArgs(named *types.Named) []types.Type { + return named.TArgs() +} + +// InitInferred initializes info to record inferred type information. +func InitInferred(info *types.Info) { + info.Inferred = make(map[ast.Expr]types.Inferred) +} + +// GetInferred extracts inferred type information from info for e. +// +// The expression e may have an inferred type if it is an *ast.IndexExpr +// representing partial instantiation of a generic function type for which type +// arguments have been inferred using constraint type inference, or if it is an +// *ast.CallExpr for which type type arguments have be inferred using both +// constraint type inference and function argument inference. +func GetInferred(info *types.Info, e ast.Expr) ([]types.Type, *types.Signature) { + if info.Inferred == nil { + return nil, nil + } + inf := info.Inferred[e] + return inf.TArgs, inf.Sig +} diff --git a/src/cmd/go/internal/txtar/archive.go b/src/cmd/vendor/golang.org/x/tools/txtar/archive.go similarity index 96% rename from src/cmd/go/internal/txtar/archive.go rename to src/cmd/vendor/golang.org/x/tools/txtar/archive.go index 17966848771..214256617b5 100644 --- a/src/cmd/go/internal/txtar/archive.go +++ b/src/cmd/vendor/golang.org/x/tools/txtar/archive.go @@ -34,7 +34,7 @@ package txtar import ( "bytes" "fmt" - "os" + "io/ioutil" "strings" ) @@ -66,7 +66,7 @@ func Format(a *Archive) []byte { // ParseFile parses the named file as an archive. func ParseFile(file string) (*Archive, error) { - data, err := os.ReadFile(file) + data, err := ioutil.ReadFile(file) if err != nil { return nil, err } @@ -121,7 +121,7 @@ func isMarker(data []byte) (name string, after []byte) { if i := bytes.IndexByte(data, '\n'); i >= 0 { data, after = data[:i], data[i+1:] } - if !bytes.HasSuffix(data, markerEnd) { + if !(bytes.HasSuffix(data, markerEnd) && len(data) >= len(marker)+len(markerEnd)) { return "", nil } return strings.TrimSpace(string(data[len(marker) : len(data)-len(markerEnd)])), after diff --git a/src/cmd/vendor/modules.txt b/src/cmd/vendor/modules.txt index beceef5392c..e032ccc2c75 100644 --- a/src/cmd/vendor/modules.txt +++ b/src/cmd/vendor/modules.txt @@ -48,7 +48,7 @@ golang.org/x/sys/windows # golang.org/x/term v0.0.0-20210503060354-a79de5458b56 ## explicit; go 1.17 golang.org/x/term -# golang.org/x/tools v0.1.2-0.20210519160823-49064d2332f9 +# golang.org/x/tools v0.1.6-0.20210726171848-ebce39e5e3d6 ## explicit; go 1.17 golang.org/x/tools/cover golang.org/x/tools/go/analysis @@ -92,6 +92,8 @@ golang.org/x/tools/go/types/objectpath golang.org/x/tools/go/types/typeutil golang.org/x/tools/internal/analysisinternal golang.org/x/tools/internal/lsp/fuzzy +golang.org/x/tools/internal/typeparams +golang.org/x/tools/txtar # golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 ## explicit; go 1.11 golang.org/x/xerrors diff --git a/src/cmd/vet/testdata/print/print.go b/src/cmd/vet/testdata/print/print.go index fca594925f7..be42a377175 100644 --- a/src/cmd/vet/testdata/print/print.go +++ b/src/cmd/vet/testdata/print/print.go @@ -491,10 +491,10 @@ type recursiveStringer int func (s recursiveStringer) String() string { _ = fmt.Sprintf("%d", s) _ = fmt.Sprintf("%#v", s) - _ = fmt.Sprintf("%v", s) // ERROR "Sprintf format %v with arg s causes recursive String method call" - _ = fmt.Sprintf("%v", &s) // ERROR "Sprintf format %v with arg &s causes recursive String method call" + _ = fmt.Sprintf("%v", s) // ERROR "Sprintf format %v with arg s causes recursive \(cmd/vet/testdata/print\.recursiveStringer\)\.String method call" + _ = fmt.Sprintf("%v", &s) // ERROR "Sprintf format %v with arg &s causes recursive \(cmd/vet/testdata/print\.recursiveStringer\)\.String method call" _ = fmt.Sprintf("%T", s) // ok; does not recursively call String - return fmt.Sprintln(s) // ERROR "Sprintln arg s causes recursive call to String method" + return fmt.Sprintln(s) // ERROR "Sprintln arg s causes recursive call to \(cmd/vet/testdata/print\.recursiveStringer\)\.String method" } type recursivePtrStringer int @@ -502,7 +502,7 @@ type recursivePtrStringer int func (p *recursivePtrStringer) String() string { _ = fmt.Sprintf("%v", *p) _ = fmt.Sprint(&p) // ok; prints address - return fmt.Sprintln(p) // ERROR "Sprintln arg p causes recursive call to String method" + return fmt.Sprintln(p) // ERROR "Sprintln arg p causes recursive call to \(\*cmd/vet/testdata/print\.recursivePtrStringer\)\.String method" } type BoolFormatter bool From 3025ce2fa83c86f3c802aab483535c39bad508ea Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Tue, 3 Aug 2021 14:51:46 -0700 Subject: [PATCH 20/25] [dev.cmdgo] cmd/go: address code review comments in test cgo_path_space_quote For CL 334732. Change-Id: I5cb88cd7d5e4edf6006bbaeb17723dac2cdf0fd5 Reviewed-on: https://go-review.googlesource.com/c/go/+/339590 Trust: Jay Conrod Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- src/cmd/go/testdata/script/cgo_path_space_quote.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cmd/go/testdata/script/cgo_path_space_quote.txt b/src/cmd/go/testdata/script/cgo_path_space_quote.txt index 3b89bfb8001..95561013008 100644 --- a/src/cmd/go/testdata/script/cgo_path_space_quote.txt +++ b/src/cmd/go/testdata/script/cgo_path_space_quote.txt @@ -6,6 +6,7 @@ [short] skip [!exec:clang] [!exec:gcc] skip +[!cgo] skip env GOENV=$WORK/go.env mkdir 'program files' @@ -20,6 +21,7 @@ go env CC stdout 'program files[/\\]which cc" (clang|gcc)$' go run . +stdout 1 -- go.mod -- module test From aaf914d0e69198a96683c106abb5a931c4956f88 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 2 Aug 2021 16:02:45 -0400 Subject: [PATCH 21/25] [dev.cmdgo] cmd/go: remove modload.ModRoot function In some cases, ModRoot was being called in a multi module context. In those cases, pass in the correct main module. In other cases, a mainModule variable was already available, so call MainModules.ModRoot on that mainModule variable to make it more clear. In yet other cases ModRoot is just needed to determine GoMod, so determine modroot from the current directory in those cases. For #45713 Change-Id: I8c8aa633cfae40d0c8740bdbf985f2b60c9daf2c Reviewed-on: https://go-review.googlesource.com/c/go/+/339171 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/envcmd/env.go | 2 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modget/get.go | 14 ++++++-- src/cmd/go/internal/modload/build.go | 2 +- src/cmd/go/internal/modload/import.go | 6 ++-- src/cmd/go/internal/modload/init.go | 48 +++++++++++--------------- src/cmd/go/internal/modload/load.go | 17 ++++++--- src/cmd/go/internal/modload/modfile.go | 2 +- src/cmd/go/internal/modload/vendor.go | 6 ++-- 9 files changed, 56 insertions(+), 43 deletions(-) diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 5c45e343309..d23d5391411 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -148,7 +148,7 @@ func ExtraEnvVars() []cfg.EnvVar { gomod := "" modload.Init() if modload.HasModRoot() { - gomod = filepath.Join(modload.ModRoot(), "go.mod") + gomod = modload.ModFilePath() } else if modload.Enabled() { gomod = os.DevNull } diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 774fc3052fb..a51ac21751a 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -74,7 +74,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { } _, pkgs := modload.LoadPackages(ctx, loadOpts, "all") - vdir := filepath.Join(modload.ModRoot(), "vendor") + vdir := filepath.Join(modload.VendorDir()) if err := os.RemoveAll(vdir); err != nil { base.Fatalf("go mod vendor: %v", err) } diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 6eae44f1a42..3d831a14d80 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -724,7 +724,16 @@ func (r *resolver) performLocalQueries(ctx context.Context) { // restricted to matching packages in the main module. pkgPattern, mainModule := modload.MainModules.DirImportPath(ctx, q.pattern) if pkgPattern == "." { - return errSet(fmt.Errorf("%s%s is not within module rooted at %s", q.pattern, absDetail, modload.ModRoot())) + modload.MustHaveModRoot() + var modRoots []string + for _, m := range modload.MainModules.Versions() { + modRoots = append(modRoots, modload.MainModules.ModRoot(m)) + } + var plural string + if len(modRoots) != 1 { + plural = "s" + } + return errSet(fmt.Errorf("%s%s is not within module%s rooted at %s", q.pattern, absDetail, plural, strings.Join(modRoots, ", "))) } match := modload.MatchInModule(ctx, pkgPattern, mainModule, imports.AnyTags()) @@ -737,7 +746,8 @@ func (r *resolver) performLocalQueries(ctx context.Context) { return errSet(fmt.Errorf("no package in current directory")) } if !q.isWildcard() { - return errSet(fmt.Errorf("%s%s is not a package in module rooted at %s", q.pattern, absDetail, modload.ModRoot())) + modload.MustHaveModRoot() + return errSet(fmt.Errorf("%s%s is not a package in module rooted at %s", q.pattern, absDetail, modload.MainModules.ModRoot(mainModule))) } search.WarnUnmatched([]*search.Match{match}) return pathSet{} diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 29735864790..3f2160d52da 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -323,7 +323,7 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li if filepath.IsAbs(r.Path) { info.Replace.Dir = r.Path } else { - info.Replace.Dir = filepath.Join(ModRoot(), r.Path) + info.Replace.Dir = filepath.Join(replacedFrom, r.Path) } info.Replace.GoMod = filepath.Join(info.Replace.Dir, "go.mod") } diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 5741299281a..de47974b9b7 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -294,7 +294,7 @@ func importFromModules(ctx context.Context, path string, rs *Requirements, mg *M if mainErr != nil { return module.Version{}, "", mainErr } - readVendorList() + readVendorList(mainModule) return vendorPkgModule[path], vendorDir, nil } @@ -653,11 +653,11 @@ func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, i if modRoot := MainModules.ModRoot(mod); modRoot != "" { return modRoot, true, nil } - if r, _ := Replacement(mod); r.Path != "" { + if r, replacedFrom := Replacement(mod); r.Path != "" { if r.Version == "" { dir = r.Path if !filepath.IsAbs(dir) { - dir = filepath.Join(ModRoot(), dir) + dir = filepath.Join(replacedFrom, dir) } // Ensure that the replacement directory actually exists: // dirInModule does not report errors for missing modules, diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 5dd946215bf..1a91b83148a 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -374,7 +374,6 @@ func Init() { } if inWorkspaceMode() { - _ = TODOWorkspaces("go.work.sum, and also allow modfetch to fall back to individual go.sums") _ = TODOWorkspaces("replaces") var err error @@ -403,8 +402,7 @@ func Init() { // // See golang.org/issue/32027. } else { - _ = TODOWorkspaces("Instead of modfile path, find modfile OR workfile path depending on mode") - modfetch.GoSumFile = strings.TrimSuffix(ModFilePath(), ".mod") + ".sum" + modfetch.GoSumFile = strings.TrimSuffix(modFilePath(modRoots[0]), ".mod") + ".sum" } } @@ -463,21 +461,8 @@ func Enabled() bool { return modRoots != nil || cfg.ModulesEnabled } -// ModRoot returns the root of the main module. -// It calls base.Fatalf if there is no main module. -func ModRoot() string { - if !HasModRoot() { - die() - } - if inWorkspaceMode() { - panic("ModRoot called in workspace mode") - } - // This is similar to MustGetSingleMainModule but we can't call that - // because MainModules may not yet exist when ModRoot is called. - if len(modRoots) != 1 { - panic("not in workspace mode but there are multiple ModRoots") - } - return modRoots[0] +func VendorDir() string { + return filepath.Join(MainModules.ModRoot(MainModules.mustGetSingleMainModule()), "vendor") } func inWorkspaceMode() bool { @@ -495,12 +480,21 @@ func HasModRoot() bool { return modRoots != nil } -// ModFilePath returns the effective path of the go.mod file. Normally, this -// "go.mod" in the directory returned by ModRoot, but the -modfile flag may -// change its location. ModFilePath calls base.Fatalf if there is no main +// MustHaveModRoot checks that a main module or main modules are present, +// and calls base.Fatalf if there are no main modules. +func MustHaveModRoot() { + Init() + if !HasModRoot() { + die() + } +} + +// ModFilePath returns the path that would be used for the go.mod +// file, if in module mode. ModFilePath calls base.Fatalf if there is no main // module, even if -modfile is set. func ModFilePath() string { - return modFilePath(ModRoot()) + MustHaveModRoot() + return modFilePath(findModuleRoot(base.Cwd())) } func modFilePath(modRoot string) string { @@ -674,7 +668,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { mainModule := MainModules.mustGetSingleMainModule() if cfg.BuildMod == "vendor" { - readVendorList() + readVendorList(mainModule) index := MainModules.Index(mainModule) modFile := MainModules.ModFile(mainModule) checkVendorConsistency(index, modFile) @@ -719,7 +713,7 @@ func CreateModFile(ctx context.Context, modPath string) { modRoot := base.Cwd() modRoots = []string{modRoot} Init() - modFilePath := ModFilePath() + modFilePath := modFilePath(modRoot) if _, err := fsys.Stat(modFilePath); err == nil { base.Fatalf("go: %s already exists", modFilePath) } @@ -1344,6 +1338,7 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) return } mainModule := MainModules.Versions()[0] + modFilePath := modFilePath(MainModules.ModRoot(mainModule)) modFile := MainModules.ModFile(mainModule) var list []*modfile.Require @@ -1383,8 +1378,7 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) } return } - gomod := ModFilePath() - if _, ok := fsys.OverlayPath(gomod); ok { + if _, ok := fsys.OverlayPath(modFilePath); ok { if dirty { base.Fatalf("go: updates to go.mod needed, but go.mod is part of the overlay specified with -overlay") } @@ -1422,7 +1416,7 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) errNoChange := errors.New("no update needed") - err = lockedfile.Transform(ModFilePath(), func(old []byte) ([]byte, error) { + err = lockedfile.Transform(modFilePath, func(old []byte) ([]byte, error) { if bytes.Equal(old, new) { // The go.mod file is already equal to new, possibly as the result of some // other process. diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index dd69e2afbfe..cb5a2d7a353 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -440,7 +440,16 @@ func matchLocalDirs(ctx context.Context, m *search.Match, rs *Requirements) { if !filepath.IsAbs(dir) { absDir = filepath.Join(base.Cwd(), dir) } - if search.InDir(absDir, cfg.GOROOTsrc) == "" && search.InDir(absDir, ModRoot()) == "" && pathInModuleCache(ctx, absDir, rs) == "" { + + modRoot := findModuleRoot(absDir) + found := false + for _, mod := range MainModules.Versions() { + if MainModules.ModRoot(mod) == modRoot { + found = true + break + } + } + if !found && search.InDir(absDir, cfg.GOROOTsrc) == "" && pathInModuleCache(ctx, absDir, rs) == "" { m.Dirs = []string{} m.AddError(fmt.Errorf("directory prefix %s outside available modules", base.ShortPath(absDir))) return @@ -513,7 +522,7 @@ func resolveLocalPackage(ctx context.Context, dir string, rs *Requirements) (str return "", fmt.Errorf("without -mod=vendor, directory %s has no package path", absDir) } - readVendorList() + readVendorList(mainModule) pkg := strings.TrimPrefix(suffix, "/vendor/") if _, ok := vendorPkgModule[pkg]; !ok { return "", fmt.Errorf("directory %s is not a package listed in vendor/modules.txt", absDir) @@ -582,10 +591,10 @@ func pathInModuleCache(ctx context.Context, dir string, rs *Requirements) string tryMod := func(m module.Version) (string, bool) { var root string var err error - if repl, _ := Replacement(m); repl.Path != "" && repl.Version == "" { + if repl, replModRoot := Replacement(m); repl.Path != "" && repl.Version == "" { root = repl.Path if !filepath.IsAbs(root) { - root = filepath.Join(ModRoot(), root) + root = filepath.Join(replModRoot, root) } } else if repl.Path != "" { root, err = modfetch.DownloadDir(repl) diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index bc5c83dffce..664fc0f91bd 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -528,7 +528,7 @@ func goModSummary(m module.Version) (*modFileSummary, error) { // For every module other than the target, // return the full list of modules from modules.txt. - readVendorList() + readVendorList(MainModules.mustGetSingleMainModule()) // We don't know what versions the vendored module actually relies on, // so assume that it requires everything. diff --git a/src/cmd/go/internal/modload/vendor.go b/src/cmd/go/internal/modload/vendor.go index b2da3783ea1..daa58880754 100644 --- a/src/cmd/go/internal/modload/vendor.go +++ b/src/cmd/go/internal/modload/vendor.go @@ -36,13 +36,13 @@ type vendorMetadata struct { } // readVendorList reads the list of vendored modules from vendor/modules.txt. -func readVendorList() { +func readVendorList(mainModule module.Version) { vendorOnce.Do(func() { vendorList = nil vendorPkgModule = make(map[string]module.Version) vendorVersion = make(map[string]string) vendorMeta = make(map[module.Version]vendorMetadata) - data, err := os.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt")) + data, err := os.ReadFile(filepath.Join(MainModules.ModRoot(mainModule), "vendor/modules.txt")) if err != nil { if !errors.Is(err, fs.ErrNotExist) { base.Fatalf("go: %s", err) @@ -136,7 +136,7 @@ func readVendorList() { // go 1.14) or at least does not contradict (go 1.13 or earlier) the // requirements and replacements listed in the main module's go.mod file. func checkVendorConsistency(index *modFileIndex, modFile *modfile.File) { - readVendorList() + readVendorList(MainModules.mustGetSingleMainModule()) pre114 := false if semver.Compare(index.goVersionV, "v1.14") < 0 { From d397fc1169aa2491bc807812105371819366deb8 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 2 Aug 2021 15:54:51 -0400 Subject: [PATCH 22/25] [dev.cmdgo] don't give command-line-arguments a module Don't associate command-line-arguments with a module. Even though the sources in the command-line-arguments package may exist within the module's packages, the command-line-arguments package is distinct from the package in the module. It has its own identity, and further, even if all the same sources are listed, build tag filtering is not applied for command-line-arguments. For #45713 Change-Id: I555752021d58ea25e65699b4959f787ea5fa2cda Reviewed-on: https://go-review.googlesource.com/c/go/+/339170 Trust: Michael Matloob Trust: Bryan C. Mills Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modload/build.go | 40 ++++--------------- .../mod_list_command_line_arguments.txt | 35 ++++++++++++++++ src/cmd/go/testdata/script/mod_outside.txt | 2 +- src/cmd/go/testdata/script/version.txt | 7 ++++ src/cmd/go/testdata/script/work.txt | 16 +++++++- 5 files changed, 65 insertions(+), 35 deletions(-) create mode 100644 src/cmd/go/testdata/script/mod_list_command_line_arguments.txt diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 3f2160d52da..73b51c117a1 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -341,15 +341,14 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li // for modules providing packages named by path and deps. path and deps must // name packages that were resolved successfully with LoadPackages. func PackageBuildInfo(path string, deps []string) string { - if isStandardImportPath(path) || !Enabled() { + if !Enabled() { return "" } - - target := mustFindModule(loaded, path, path) + target, _ := findModule(loaded, path) mdeps := make(map[module.Version]bool) for _, dep := range deps { - if !isStandardImportPath(dep) { - mdeps[mustFindModule(loaded, path, dep)] = true + if m, ok := findModule(loaded, dep); ok { + mdeps[m] = true } } var mods []module.Version @@ -375,7 +374,9 @@ func PackageBuildInfo(path string, deps []string) string { } } - writeEntry("mod", target) + if target.Path != "" { + writeEntry("mod", target) + } for _, mod := range mods { writeEntry("dep", mod) } @@ -383,29 +384,6 @@ func PackageBuildInfo(path string, deps []string) string { return buf.String() } -// mustFindModule is like findModule, but it calls base.Fatalf if the -// module can't be found. -// -// TODO(jayconrod): remove this. Callers should use findModule and return -// errors instead of relying on base.Fatalf. -func mustFindModule(ld *loader, target, path string) module.Version { - pkg, ok := ld.pkgCache.Get(path).(*loadPkg) - if ok { - if pkg.err != nil { - base.Fatalf("build %v: cannot load %v: %v", target, path, pkg.err) - } - return pkg.mod - } - - if path == "command-line-arguments" { - _ = TODOWorkspaces("support multiple main modules; search by modroot") - return MainModules.mustGetSingleMainModule() - } - - base.Fatalf("build %v: cannot find module for path %v", target, path) - panic("unreachable") -} - // findModule searches for the module that contains the package at path. // If the package was loaded, its containing module and true are returned. // Otherwise, module.Version{} and false are returned. @@ -413,10 +391,6 @@ func findModule(ld *loader, path string) (module.Version, bool) { if pkg, ok := ld.pkgCache.Get(path).(*loadPkg); ok { return pkg.mod, pkg.mod != module.Version{} } - if path == "command-line-arguments" { - _ = TODOWorkspaces("support multiple main modules; search by modroot") - return MainModules.mustGetSingleMainModule(), true - } return module.Version{}, false } diff --git a/src/cmd/go/testdata/script/mod_list_command_line_arguments.txt b/src/cmd/go/testdata/script/mod_list_command_line_arguments.txt new file mode 100644 index 00000000000..fd99ae84b2e --- /dev/null +++ b/src/cmd/go/testdata/script/mod_list_command_line_arguments.txt @@ -0,0 +1,35 @@ +# The command-line-arguments package does not belong to a module... +cd a +go list -f '{{.Module}}' ../b/b.go +stdout '^$' + +# ... even if the arguments are sources from that module +go list -f '{{.Module}}' a.go +stdout '^$' + +[short] skip + +# check that the version of command-line-arguments doesn't include a module +go build -o a.exe a.go +go version -m a.exe +stdout '^\tpath\tcommand-line-arguments$' +stdout '^\tdep\ta\t\(devel\)\t$' +! stdout mod + +-- a/go.mod -- +module a +go 1.17 +-- a/a.go -- +package main + +import "a/dep" + +func main() { + dep.D() +} +-- a/dep/dep.go -- +package dep + +func D() {} +-- b/b.go -- +package b \ No newline at end of file diff --git a/src/cmd/go/testdata/script/mod_outside.txt b/src/cmd/go/testdata/script/mod_outside.txt index 33341f7d4b3..3b4559405ac 100644 --- a/src/cmd/go/testdata/script/mod_outside.txt +++ b/src/cmd/go/testdata/script/mod_outside.txt @@ -251,7 +251,7 @@ stdout 'using example.com/version v1.0.1' # outside std. go run ./stdonly/stdonly.go stdout 'path is command-line-arguments$' -stdout 'main is command-line-arguments \(devel\)' +stdout 'main is $' # 'go generate' should work with file arguments. [exec:touch] go generate ./needmod/needmod.go diff --git a/src/cmd/go/testdata/script/version.txt b/src/cmd/go/testdata/script/version.txt index 8615a4aac59..f3aa57e8c70 100644 --- a/src/cmd/go/testdata/script/version.txt +++ b/src/cmd/go/testdata/script/version.txt @@ -28,6 +28,13 @@ go version -m fortune.exe stdout '^\tpath\trsc.io/fortune' stdout '^\tmod\trsc.io/fortune\tv1.0.0' +# Check the build info of a binary built from $GOROOT/src/cmd +go build -o test2json.exe cmd/test2json +go version -m test2json.exe +stdout '^test2json.exe: .+' +stdout '^\tpath\tcmd/test2json$' +! stdout 'mod' + # Repeat the test with -buildmode=pie. [!buildmode:pie] stop go build -buildmode=pie -o external.exe rsc.io/fortune diff --git a/src/cmd/go/testdata/script/work.txt b/src/cmd/go/testdata/script/work.txt index 9be09585796..095d6ff1745 100644 --- a/src/cmd/go/testdata/script/work.txt +++ b/src/cmd/go/testdata/script/work.txt @@ -42,6 +42,13 @@ go run example.com/d cp go.work.backwards go.work go run example.com/d +# Test that command-line-arguments work inside and outside modules. +# This exercises the code that determines which module command-line-arguments +# belongs to. +go list ./b/main.go +go build -n -workfile=off -o foo foo.go +go build -n -o foo foo.go + -- go.work.dup -- go 1.17 @@ -123,4 +130,11 @@ directory ( d b a -) \ No newline at end of file +) + +-- foo.go -- +package main +import "fmt" +func main() { + fmt.Println("Hello, World") +} From e2e1987b31a587bdb67856954ae9279721b3bba7 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 12 Aug 2021 14:33:58 -0700 Subject: [PATCH 23/25] [dev.cmdgo] cmd/link: fix TestBuildForTvOS This test was broken in CL 334732 on darwin. The test invokes 'go build' with a CC containing the arguments -framework CoreFoundation. Previously, the go command split CC on whitespace, and inserted the arguments after the command line when running CC directly. Those arguments weren't passed to cgo though, so cgo ran CC without -framework CoreFoundation (or any of the other flags). In CL 334732, we pass CC through to cgo, and cgo splits arguments using str.SplitQuotedFields. So -framework CoreFoundation actually gets passed to the C compiler. It appears that -framework flags are only meant to be used in linking operations, so when cgo invokes clang with -E (run preprocessor only), clang emits an error that -framework is unused. This change fixes the test by moving -framework CoreFoundation out of CC and into CGO_LDFLAGS. Change-Id: Ie884c3c0d8bea21fad57f325d19989ad39de7204 Reviewed-on: https://go-review.googlesource.com/c/go/+/341929 Run-TryBot: Jay Conrod TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills Trust: Jay Conrod --- src/cmd/link/link_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cmd/link/link_test.go b/src/cmd/link/link_test.go index 7230054bedd..77d42cceda8 100644 --- a/src/cmd/link/link_test.go +++ b/src/cmd/link/link_test.go @@ -282,8 +282,8 @@ func TestBuildForTvOS(t *testing.T) { "-isysroot", strings.TrimSpace(string(sdkPath)), "-mtvos-version-min=12.0", "-fembed-bitcode", - "-framework", "CoreFoundation", } + CGO_LDFLAGS := []string{"-framework", "CoreFoundation"} lib := filepath.Join("testdata", "testBuildFortvOS", "lib.go") tmpDir := t.TempDir() @@ -295,12 +295,14 @@ func TestBuildForTvOS(t *testing.T) { "GOARCH=arm64", "CC="+strings.Join(CC, " "), "CGO_CFLAGS=", // ensure CGO_CFLAGS does not contain any flags. Issue #35459 + "CGO_LDFLAGS="+strings.Join(CGO_LDFLAGS, " "), ) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("%v: %v:\n%s", cmd.Args, err, out) } link := exec.Command(CC[0], CC[1:]...) + link.Args = append(link.Args, CGO_LDFLAGS...) link.Args = append(link.Args, "-o", filepath.Join(tmpDir, "a.out")) // Avoid writing to package directory. link.Args = append(link.Args, ar, filepath.Join("testdata", "testBuildFortvOS", "main.m")) if out, err := link.CombinedOutput(); err != nil { From 3b523caf4145c2d915c5ead69440f9b890634587 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 23 Aug 2021 14:51:39 -0400 Subject: [PATCH 24/25] [dev.cmdgo] cmd/go: clean up TODOWorkspaces instances Address some of the easier todos to address and remove the todos that have already been done and redundant todos. For #45713 Change-Id: I3fe4393168b10c6e005325258d9701713c92e9e4 Reviewed-on: https://go-review.googlesource.com/c/go/+/344491 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modcmd/download.go | 2 +- src/cmd/go/internal/modcmd/initwork.go | 4 +- src/cmd/go/internal/modget/get.go | 20 ++++----- src/cmd/go/internal/modload/build.go | 1 - src/cmd/go/internal/modload/buildlist.go | 8 ++-- src/cmd/go/internal/modload/init.go | 48 +++++++++------------ src/cmd/go/internal/modload/load.go | 3 +- src/cmd/go/internal/modload/modfile.go | 2 +- src/cmd/go/internal/modload/query.go | 53 ++++++++++++++---------- 9 files changed, 71 insertions(+), 70 deletions(-) diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 6a99cb01e1e..ff56d05116f 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -97,7 +97,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { modload.LoadModFile(ctx) // to fill MainModules if len(modload.MainModules.Versions()) != 1 { - panic(modload.TODOWorkspaces("TODO: multiple main modules not supported in Download")) + panic(modload.TODOWorkspaces("Support workspace mode in go mod download")) } mainModule := modload.MainModules.Versions()[0] diff --git a/src/cmd/go/internal/modcmd/initwork.go b/src/cmd/go/internal/modcmd/initwork.go index 30653503bcd..4182aa071d7 100644 --- a/src/cmd/go/internal/modcmd/initwork.go +++ b/src/cmd/go/internal/modcmd/initwork.go @@ -13,9 +13,9 @@ import ( "path/filepath" ) -var _ = modload.TODOWorkspaces("Add more documentation below.T hough this is" + +var _ = modload.TODOWorkspaces("Add more documentation below. Though this is" + "enough for those trying workspaces out, there should be more through" + - "documentation if the proposal is accepted.") + "documentation if the proposal is accepted and released.") var cmdInitwork = &base.Command{ UsageLine: "go mod initwork [moddirs]", diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 3d831a14d80..37912ce8334 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -690,7 +690,7 @@ func (r *resolver) queryNone(ctx context.Context, q *query) { // However, neither of those behaviors would be consistent with the // plain meaning of the query. To try to reduce confusion, reject the // query explicitly. - return errSet(&modload.QueryMatchesMainModuleError{MainModule: v, Pattern: q.pattern, Query: q.version}) + return errSet(&modload.QueryMatchesMainModulesError{MainModules: []module.Version{v}, Pattern: q.pattern, Query: q.version}) } return pathSet{mod: module.Version{Path: q.pattern, Version: "none"}} @@ -703,7 +703,7 @@ func (r *resolver) queryNone(ctx context.Context, q *query) { } q.pathOnce(curM.Path, func() pathSet { if modload.HasModRoot() && curM.Version == "" && modload.MainModules.Contains(curM.Path) { - return errSet(&modload.QueryMatchesMainModuleError{MainModule: curM, Pattern: q.pattern, Query: q.version}) + return errSet(&modload.QueryMatchesMainModulesError{MainModules: []module.Version{curM}, Pattern: q.pattern, Query: q.version}) } return pathSet{mod: module.Version{Path: curM.Path, Version: "none"}} }) @@ -805,10 +805,10 @@ func (r *resolver) queryWildcard(ctx context.Context, q *query) { if modload.MainModules.Contains(curM.Path) && !versionOkForMainModule(q.version) { if q.matchesPath(curM.Path) { - return errSet(&modload.QueryMatchesMainModuleError{ - MainModule: curM, - Pattern: q.pattern, - Query: q.version, + return errSet(&modload.QueryMatchesMainModulesError{ + MainModules: []module.Version{curM}, + Pattern: q.pattern, + Query: q.version, }) } @@ -1760,10 +1760,10 @@ func (r *resolver) resolve(q *query, m module.Version) { } if modload.MainModules.Contains(m.Path) && m.Version != "" { - reportError(q, &modload.QueryMatchesMainModuleError{ - MainModule: module.Version{Path: m.Path}, - Pattern: q.pattern, - Query: q.version, + reportError(q, &modload.QueryMatchesMainModulesError{ + MainModules: []module.Version{{Path: m.Path}}, + Pattern: q.pattern, + Query: q.version, }) return } diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 73b51c117a1..0efd84123af 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -218,7 +218,6 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li Version: m.Version, Main: true, } - _ = TODOWorkspaces("handle rawGoVersion here") if v, ok := rawGoVersion.Load(m); ok { info.GoVersion = v.(string) } else { diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 9989bb5b2a4..14379b4c3c1 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -139,7 +139,7 @@ func (rs *Requirements) initVendor(vendorList []module.Version) { } if MainModules.Len() != 1 { - panic("There should be exactly one main moudle in Vendor mode.") + panic("There should be exactly one main module in Vendor mode.") } mainModule := MainModules.Versions()[0] @@ -284,8 +284,10 @@ func readModGraph(ctx context.Context, depth modDepth, roots []module.Version) ( ) for _, m := range MainModules.Versions() { // Require all roots from all main modules. - _ = TODOWorkspaces("This isn't the correct behavior. " + - "Fix this when the requirements struct is updated to reflect the struct of the module graph.") + _ = TODOWorkspaces("This flattens a level of the module graph, adding the dependencies " + + "of all main modules to a single requirements struct, and losing the information of which " + + "main module required which requirement. Rework the requirements struct and change this" + + "to reflect the structure of the main modules.") mg.g.Require(m, roots) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 896c61d19dd..ab6733830fc 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -123,8 +123,6 @@ func (mms *MainModuleSet) Contains(path string) bool { } func (mms *MainModuleSet) ModRoot(m module.Version) string { - _ = TODOWorkspaces(" Do we need the Init? The original modRoot calls it. Audit callers.") - Init() if mms == nil { return "" } @@ -143,8 +141,11 @@ func (mms *MainModuleSet) mustGetSingleMainModule() module.Version { panic("internal error: mustGetSingleMainModule called in context with no main modules") } if len(mms.versions) != 1 { - _ = TODOWorkspaces("Check if we're in workspace mode before returning the below error.") - panic("internal error: mustGetSingleMainModule called in workspace mode") + if inWorkspaceMode() { + panic("internal error: mustGetSingleMainModule called in workspace mode") + } else { + panic("internal error: multiple main modules present outside of workspace mode") + } } return mms.versions[0] } @@ -156,11 +157,7 @@ func (mms *MainModuleSet) GetSingleIndexOrNil() *modFileIndex { if len(mms.versions) == 0 { return nil } - if len(mms.versions) != 1 { - _ = TODOWorkspaces("Check if we're in workspace mode before returning the below error.") - panic("internal error: mustGetSingleMainModule called in workspace mode") - } - return mms.indices[mms.versions[0]] + return mms.indices[mms.mustGetSingleMainModule()] } func (mms *MainModuleSet) Index(m module.Version) *modFileIndex { @@ -363,7 +360,9 @@ func Init() { // We're in module mode. Set any global variables that need to be set. cfg.ModulesEnabled = true setDefaultBuildMod() - _ = TODOWorkspaces("ensure that buildmod is readonly") + _ = TODOWorkspaces("In workspace mode, mod will not be readonly for go mod download," + + "verify, graph, and why. Implement support for go mod download and add test cases" + + "to ensure verify, graph, and why work properly.") list := filepath.SplitList(cfg.BuildContext.GOPATH) if len(list) == 0 || list[0] == "" { base.Fatalf("missing $GOPATH") @@ -374,15 +373,14 @@ func Init() { } if inWorkspaceMode() { - _ = TODOWorkspaces("go.work.sum, and also allow modfetch to fall back to individual go.sums") - _ = TODOWorkspaces("replaces") var err error modRoots, err = loadWorkFile(workFilePath) if err != nil { base.Fatalf("reading go.work: %v", err) } + _ = TODOWorkspaces("Support falling back to individual module go.sum " + + "files for sums not in the workspace sum file.") modfetch.GoSumFile = workFilePath + ".sum" - // TODO(matloob) should workRoot just be workFile? } else if modRoots == nil { // We're in module mode, but not inside a module. // @@ -539,6 +537,7 @@ func (goModDirtyError) Error() string { var errGoModDirty error = goModDirtyError{} func loadWorkFile(path string) (modRoots []string, err error) { + _ = TODOWorkspaces("Clean up and write back the go.work file: add module paths for workspace modules.") workDir := filepath.Dir(path) workData, err := lockedfile.Read(path) if err != nil { @@ -661,8 +660,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { // We don't need to do anything for vendor or update the mod file so // return early. - _ = TODOWorkspaces("don't worry about commits for now, but eventually will want to update go.work files") - return rs, false + return rs, true } mainModule := MainModules.mustGetSingleMainModule() @@ -761,7 +759,7 @@ func CreateModFile(ctx context.Context, modPath string) { MainModules = makeMainModules([]module.Version{modFile.Module.Mod}, []string{modRoot}, []*modfile.File{modFile}, []*modFileIndex{nil}) addGoStmt(modFile, modFile.Module.Mod, LatestGoVersion()) // Add the go directive before converted module requirements. - convertedFrom, err := convertLegacyConfig(modFile, modPath) + convertedFrom, err := convertLegacyConfig(modFile, modRoot) if convertedFrom != "" { fmt.Fprintf(os.Stderr, "go: copying requirements from %s\n", base.ShortPath(convertedFrom)) } @@ -1037,7 +1035,7 @@ func mustHaveCompleteRequirements() bool { // convertLegacyConfig imports module requirements from a legacy vendoring // configuration file, if one is present. -func convertLegacyConfig(modFile *modfile.File, modPath string) (from string, err error) { +func convertLegacyConfig(modFile *modfile.File, modRoot string) (from string, err error) { noneSelected := func(path string) (version string) { return "none" } queryPackage := func(path, rev string) (module.Version, error) { pkgMods, modOnly, err := QueryPattern(context.Background(), path, rev, noneSelected, nil) @@ -1050,10 +1048,7 @@ func convertLegacyConfig(modFile *modfile.File, modPath string) (from string, er return modOnly.Mod, nil } for _, name := range altConfigs { - if len(modRoots) != 1 { - panic(TODOWorkspaces("what do do here?")) - } - cfg := filepath.Join(modRoots[0], name) + cfg := filepath.Join(modRoot, name) data, err := os.ReadFile(cfg) if err == nil { convert := modconv.Converters[name] @@ -1166,7 +1161,8 @@ func findWorkspaceFile(dir string) (root string) { break } if d == cfg.GOROOT { - _ = TODOWorkspaces("Address how go.work files interact with GOROOT") + _ = TODOWorkspaces("If we end up checking in a go.work file to GOROOT/src," + + "remove this case.") return "" // As a special case, don't cross GOROOT to find a go.work file. } dir = d @@ -1345,7 +1341,7 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) // We aren't in a module, so we don't have anywhere to write a go.mod file. return } - mainModule := MainModules.Versions()[0] + mainModule := MainModules.mustGetSingleMainModule() modFilePath := modFilePath(MainModules.ModRoot(mainModule)) modFile := MainModules.ModFile(mainModule) @@ -1398,12 +1394,6 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) base.Fatalf("go: %v", err) } defer func() { - if MainModules.Len() != 1 { - panic(TODOWorkspaces("There should be exactly one main module when committing reqs")) - } - - mainModule := MainModules.Versions()[0] - // At this point we have determined to make the go.mod file on disk equal to new. MainModules.SetIndex(mainModule, indexModFile(new, modFile, mainModule, false)) diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index cb5a2d7a353..c9004ff7964 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -552,8 +552,7 @@ func resolveLocalPackage(ctx context.Context, dir string, rs *Requirements) (str // return an error. if len(mainModulePrefix) > len(pkgNotFoundLongestPrefix) { pkgNotFoundLongestPrefix = mainModulePrefix - pkgNotFoundErr = &PackageNotInModuleError{Mod: mainModule, Pattern: pkg} - + pkgNotFoundErr = &PackageNotInModuleError{MainModules: []module.Version{mainModule}, Pattern: pkg} } continue } diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 664fc0f91bd..09e9c67659e 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -322,7 +322,7 @@ func replacement(mod module.Version, index *modFileIndex) (fromVersion string, t // If there is no replacement for mod, Replacement returns // a module.Version with Path == "". func Replacement(mod module.Version) (module.Version, string) { - _ = TODOWorkspaces("support replaces in the go.work file") + _ = TODOWorkspaces("Support replaces in the go.work file.") foundFrom, found, foundModRoot := "", module.Version{}, "" for _, v := range MainModules.Versions() { if index := MainModules.Index(v); index != nil { diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 6d6bfe774cd..82979fbda12 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -632,18 +632,16 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin if modOnly != nil { return nil, modOnly, nil } else if len(mainModuleMatches) != 0 { - _ = TODOWorkspaces("add multiple main modules to the error?") - return nil, nil, &QueryMatchesMainModuleError{ - MainModule: mainModuleMatches[0], - Pattern: pattern, - Query: query, + return nil, nil, &QueryMatchesMainModulesError{ + MainModules: mainModuleMatches, + Pattern: pattern, + Query: query, } } else { - _ = TODOWorkspaces("This should maybe be PackageNotInModule*s* error with the main modules that are prefixes of base") return nil, nil, &PackageNotInModuleError{ - Mod: MainModules.Versions()[0], - Query: query, - Pattern: pattern, + MainModules: mainModuleMatches, + Query: query, + Pattern: pattern, } } } @@ -695,7 +693,7 @@ func QueryPattern(ctx context.Context, pattern, query string, current func(strin }) if len(mainModuleMatches) > 0 && len(results) == 0 && modOnly == nil && errors.Is(err, fs.ErrNotExist) { - return nil, nil, &QueryMatchesMainModuleError{ + return nil, nil, &QueryMatchesMainModulesError{ Pattern: pattern, Query: query, } @@ -893,6 +891,7 @@ func (e *WildcardInFirstElementError) Error() string { // code for the versions it knows about, and thus did not have the opportunity // to return a non-400 status code to suppress fallback. type PackageNotInModuleError struct { + MainModules []module.Version Mod module.Version Replacement module.Version Query string @@ -900,11 +899,15 @@ type PackageNotInModuleError struct { } func (e *PackageNotInModuleError) Error() string { - if MainModules.Contains(e.Mod.Path) { - if strings.Contains(e.Pattern, "...") { - return fmt.Sprintf("main module (%s) does not contain packages matching %s", e.Mod.Path, e.Pattern) + if len(e.MainModules) > 0 { + prefix := "workspace modules do" + if len(e.MainModules) == 1 { + prefix = fmt.Sprintf("main module (%s) does", e.MainModules[0]) } - return fmt.Sprintf("main module (%s) does not contain package %s", e.Mod.Path, e.Pattern) + if strings.Contains(e.Pattern, "...") { + return fmt.Sprintf("%s not contain packages matching %s", prefix, e.Pattern) + } + return fmt.Sprintf("%s not contain package %s", prefix, e.Pattern) } found := "" @@ -1153,21 +1156,29 @@ func (rr *replacementRepo) replacementStat(v string) (*modfetch.RevInfo, error) return rev, nil } -// A QueryMatchesMainModuleError indicates that a query requests +// A QueryMatchesMainModulesError indicates that a query requests // a version of the main module that cannot be satisfied. // (The main module's version cannot be changed.) -type QueryMatchesMainModuleError struct { - MainModule module.Version - Pattern string - Query string +type QueryMatchesMainModulesError struct { + MainModules []module.Version + Pattern string + Query string } -func (e *QueryMatchesMainModuleError) Error() string { +func (e *QueryMatchesMainModulesError) Error() string { if MainModules.Contains(e.Pattern) { return fmt.Sprintf("can't request version %q of the main module (%s)", e.Query, e.Pattern) } - return fmt.Sprintf("can't request version %q of pattern %q that includes the main module (%s)", e.Query, e.Pattern, e.MainModule.Path) + plural := "" + mainModulePaths := make([]string, len(e.MainModules)) + for i := range e.MainModules { + mainModulePaths[i] = e.MainModules[i].Path + } + if len(e.MainModules) > 1 { + plural = "s" + } + return fmt.Sprintf("can't request version %q of pattern %q that includes the main module%s (%s)", e.Query, e.Pattern, plural, strings.Join(mainModulePaths, ", ")) } // A QueryUpgradesAllError indicates that a query requests From de23549a3967ade982d848a5b6ae3cb3fa0dba45 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 24 Aug 2021 12:37:15 -0400 Subject: [PATCH 25/25] [dev.cmdgo] cmd/go: fix calls to modFileGoVersion to pass in modFile Before this change, we were arbitrarily picking a module to get the Go version from in calls to modFileGoVersion. We now pass in the modFile to modFileGoVersion when we have the file. Most of the calls were to get the goVersion argument for commitRequirements, so now we have commitRequirements call modFileGoVersion on the modFile directly One of the calls to commitRequirements (when running go mod tidy with a different Go version) passed in a new go version to update the file to. Now, the modFile is updated before calling commitRequirements. For the remaining cases of modFileGoVersion, it's replaced by a call to the new (*MainModuleSet).GoVersion function, which either returns the go version on the workspace file (in workspace mode) or the version of the single go.mod file. Change-Id: Ie88c3ca76c7f29ffc4faa16bb76f6cb7eccb5029 Reviewed-on: https://go-review.googlesource.com/c/go/+/344749 Trust: Michael Matloob Run-TryBot: Michael Matloob TryBot-Result: Go Bot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modload/buildlist.go | 4 +- src/cmd/go/internal/modload/init.go | 76 +++++++++++++++--------- src/cmd/go/internal/modload/list.go | 2 +- src/cmd/go/internal/modload/load.go | 14 +++-- src/cmd/go/internal/modload/modfile.go | 8 +-- 5 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/cmd/go/internal/modload/buildlist.go b/src/cmd/go/internal/modload/buildlist.go index 14379b4c3c1..94414278abb 100644 --- a/src/cmd/go/internal/modload/buildlist.go +++ b/src/cmd/go/internal/modload/buildlist.go @@ -461,7 +461,7 @@ func LoadModGraph(ctx context.Context, goVersion string) *ModuleGraph { base.Fatalf("go: %v", err) } - commitRequirements(ctx, modFileGoVersion(), rs) + commitRequirements(ctx, rs) return mg } @@ -527,7 +527,7 @@ func EditBuildList(ctx context.Context, add, mustSelect []module.Version) (chang if err != nil { return false, err } - commitRequirements(ctx, modFileGoVersion(), rs) + commitRequirements(ctx, rs) return changed, err } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index ab6733830fc..b845842a7fa 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -59,8 +59,9 @@ var ( // roots are required but MainModules hasn't been initialized yet. Set to // the modRoots of the main modules. // modRoots != nil implies len(modRoots) > 0 - modRoots []string - gopath string + modRoots []string + gopath string + workFileGoVersion string ) // Variable set in InitWorkfile @@ -91,6 +92,8 @@ type MainModuleSet struct { modContainingCWD module.Version + workFileGoVersion string + indexMu sync.Mutex indices map[module.Version]*modFileIndex } @@ -190,6 +193,20 @@ func (mms *MainModuleSet) ModContainingCWD() module.Version { return mms.modContainingCWD } +// GoVersion returns the go version set on the single module, in module mode, +// or the go.work file in workspace mode. +func (mms *MainModuleSet) GoVersion() string { + if !inWorkspaceMode() { + return modFileGoVersion(mms.ModFile(mms.mustGetSingleMainModule())) + } + v := mms.workFileGoVersion + if v == "" { + // Fall back to 1.18 for go.work files. + v = "1.18" + } + return v +} + var MainModules *MainModuleSet type Root int @@ -374,7 +391,7 @@ func Init() { if inWorkspaceMode() { var err error - modRoots, err = loadWorkFile(workFilePath) + workFileGoVersion, modRoots, err = loadWorkFile(workFilePath) if err != nil { base.Fatalf("reading go.work: %v", err) } @@ -536,16 +553,19 @@ func (goModDirtyError) Error() string { var errGoModDirty error = goModDirtyError{} -func loadWorkFile(path string) (modRoots []string, err error) { +func loadWorkFile(path string) (goVersion string, modRoots []string, err error) { _ = TODOWorkspaces("Clean up and write back the go.work file: add module paths for workspace modules.") workDir := filepath.Dir(path) workData, err := lockedfile.Read(path) if err != nil { - return nil, err + return "", nil, err } wf, err := modfile.ParseWork(path, workData, nil) if err != nil { - return nil, err + return "", nil, err + } + if wf.Go != nil { + goVersion = wf.Go.Version } seen := map[string]bool{} for _, d := range wf.Directory { @@ -554,12 +574,12 @@ func loadWorkFile(path string) (modRoots []string, err error) { modRoot = filepath.Join(workDir, modRoot) } if seen[modRoot] { - return nil, fmt.Errorf("path %s appears multiple times in workspace", modRoot) + return "", nil, fmt.Errorf("path %s appears multiple times in workspace", modRoot) } seen[modRoot] = true modRoots = append(modRoots, modRoot) } - return modRoots, nil + return goVersion, modRoots, nil } // LoadModFile sets Target and, if there is a main module, parses the initial @@ -582,7 +602,7 @@ func loadWorkFile(path string) (modRoots []string, err error) { func LoadModFile(ctx context.Context) *Requirements { rs, needCommit := loadModFile(ctx) if needCommit { - commitRequirements(ctx, modFileGoVersion(), rs) + commitRequirements(ctx, rs) } return rs } @@ -602,7 +622,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { if len(modRoots) == 0 { _ = TODOWorkspaces("Instead of creating a fake module with an empty modroot, make MainModules.Len() == 0 mean that we're in module mode but not inside any module.") mainModule := module.Version{Path: "command-line-arguments"} - MainModules = makeMainModules([]module.Version{mainModule}, []string{""}, []*modfile.File{nil}, []*modFileIndex{nil}) + MainModules = makeMainModules([]module.Version{mainModule}, []string{""}, []*modfile.File{nil}, []*modFileIndex{nil}, "") goVersion := LatestGoVersion() rawGoVersion.Store(mainModule, goVersion) requirements = newRequirements(modDepthFromGoVersion(goVersion), nil, nil) @@ -652,7 +672,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { } } - MainModules = makeMainModules(mainModules, modRoots, modFiles, indices) + MainModules = makeMainModules(mainModules, modRoots, modFiles, indices, workFileGoVersion) setDefaultBuildMod() // possibly enable automatic vendoring rs = requirementsFromModFiles(ctx, modFiles) @@ -702,7 +722,7 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) { } } } else { - rawGoVersion.Store(mainModule, modFileGoVersion()) + rawGoVersion.Store(mainModule, modFileGoVersion(MainModules.ModFile(mainModule))) } } @@ -756,7 +776,7 @@ func CreateModFile(ctx context.Context, modPath string) { fmt.Fprintf(os.Stderr, "go: creating new go.mod: module %s\n", modPath) modFile := new(modfile.File) modFile.AddModuleStmt(modPath) - MainModules = makeMainModules([]module.Version{modFile.Module.Mod}, []string{modRoot}, []*modfile.File{modFile}, []*modFileIndex{nil}) + MainModules = makeMainModules([]module.Version{modFile.Module.Mod}, []string{modRoot}, []*modfile.File{modFile}, []*modFileIndex{nil}, "") addGoStmt(modFile, modFile.Module.Mod, LatestGoVersion()) // Add the go directive before converted module requirements. convertedFrom, err := convertLegacyConfig(modFile, modRoot) @@ -772,7 +792,7 @@ func CreateModFile(ctx context.Context, modPath string) { if err != nil { base.Fatalf("go: %v", err) } - commitRequirements(ctx, modFileGoVersion(), rs) + commitRequirements(ctx, rs) // Suggest running 'go mod tidy' unless the project is empty. Even if we // imported all the correct requirements above, we're probably missing @@ -880,7 +900,7 @@ func AllowMissingModuleImports() { // makeMainModules creates a MainModuleSet and associated variables according to // the given main modules. -func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile.File, indices []*modFileIndex) *MainModuleSet { +func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile.File, indices []*modFileIndex, workFileGoVersion string) *MainModuleSet { for _, m := range ms { if m.Version != "" { panic("mainModulesCalled with module.Version with non empty Version field: " + fmt.Sprintf("%#v", m)) @@ -888,12 +908,13 @@ func makeMainModules(ms []module.Version, rootDirs []string, modFiles []*modfile } modRootContainingCWD := findModuleRoot(base.Cwd()) mainModules := &MainModuleSet{ - versions: ms[:len(ms):len(ms)], - inGorootSrc: map[module.Version]bool{}, - pathPrefix: map[module.Version]string{}, - modRoot: map[module.Version]string{}, - modFiles: map[module.Version]*modfile.File{}, - indices: map[module.Version]*modFileIndex{}, + versions: ms[:len(ms):len(ms)], + inGorootSrc: map[module.Version]bool{}, + pathPrefix: map[module.Version]string{}, + modRoot: map[module.Version]string{}, + modFiles: map[module.Version]*modfile.File{}, + indices: map[module.Version]*modFileIndex{}, + workFileGoVersion: workFileGoVersion, } for i, m := range ms { mainModules.pathPrefix[m] = m.Path @@ -958,7 +979,7 @@ func requirementsFromModFiles(ctx context.Context, modFiles []*modfile.File) *Re } } module.Sort(roots) - rs := newRequirements(modDepthFromGoVersion(modFileGoVersion()), roots, direct) + rs := newRequirements(modDepthFromGoVersion(MainModules.GoVersion()), roots, direct) return rs } @@ -1315,12 +1336,13 @@ func WriteGoMod(ctx context.Context) { if !allowWriteGoMod { panic("WriteGoMod called while disallowed") } - commitRequirements(ctx, modFileGoVersion(), LoadModFile(ctx)) + commitRequirements(ctx, LoadModFile(ctx)) } // commitRequirements writes sets the global requirements variable to rs and // writes its contents back to the go.mod file on disk. -func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) { +// goVersion, if non-empty, is used to set the version on the go.mod file. +func commitRequirements(ctx context.Context, rs *Requirements) { requirements = rs if !allowWriteGoMod { @@ -1352,10 +1374,10 @@ func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) Indirect: !rs.direct[m.Path], }) } - if goVersion != "" { - modFile.AddGoStmt(goVersion) + if modFile.Go == nil || modFile.Go.Version == "" { + modFile.AddGoStmt(modFileGoVersion(modFile)) } - if semver.Compare("v"+modFileGoVersion(), separateIndirectVersionV) < 0 { + if semver.Compare("v"+modFileGoVersion(modFile), separateIndirectVersionV) < 0 { modFile.SetRequire(list) } else { modFile.SetRequireSeparateIndirect(list) diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 1862bef4945..9c5018f340b 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -72,7 +72,7 @@ func ListModules(ctx context.Context, args []string, mode ListMode) ([]*modinfo. } if err == nil { - commitRequirements(ctx, modFileGoVersion(), rs) + commitRequirements(ctx, rs) } return mods, err } diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index c9004ff7964..efe6ad1319c 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -407,11 +407,17 @@ func LoadPackages(ctx context.Context, opts PackageOpts, patterns ...string) (ma base.Fatalf("go: %v", err) } } + + // Update the go.mod file's Go version if necessary. + modFile := MainModules.ModFile(MainModules.mustGetSingleMainModule()) + if ld.GoVersion != "" { + modFile.AddGoStmt(ld.GoVersion) + } } // Success! Update go.mod and go.sum (if needed) and return the results. loaded = ld - commitRequirements(ctx, loaded.GoVersion, loaded.requirements) + commitRequirements(ctx, loaded.requirements) for _, pkg := range ld.pkgs { if !pkg.isTest() { @@ -678,7 +684,7 @@ func ImportFromFiles(ctx context.Context, gofiles []string) { return roots }, }) - commitRequirements(ctx, loaded.GoVersion, loaded.requirements) + commitRequirements(ctx, loaded.requirements) } // DirImportPath returns the effective import path for dir, @@ -960,7 +966,7 @@ func loadFromRoots(ctx context.Context, params loaderParams) *loader { } if ld.GoVersion == "" { - ld.GoVersion = modFileGoVersion() + ld.GoVersion = MainModules.GoVersion() if ld.Tidy && semver.Compare("v"+ld.GoVersion, "v"+LatestGoVersion()) > 0 { ld.errorf("go mod tidy: go.mod file indicates go %s, but maximum supported version is %s\n", ld.GoVersion, LatestGoVersion()) @@ -1836,7 +1842,7 @@ func (ld *loader) checkTidyCompatibility(ctx context.Context, rs *Requirements) fmt.Fprintln(os.Stderr) goFlag := "" - if ld.GoVersion != modFileGoVersion() { + if ld.GoVersion != MainModules.GoVersion() { goFlag = " -go=" + ld.GoVersion } diff --git a/src/cmd/go/internal/modload/modfile.go b/src/cmd/go/internal/modload/modfile.go index 09e9c67659e..463869910c8 100644 --- a/src/cmd/go/internal/modload/modfile.go +++ b/src/cmd/go/internal/modload/modfile.go @@ -57,12 +57,8 @@ const ( ) // modFileGoVersion returns the (non-empty) Go version at which the requirements -// in modFile are intepreted, or the latest Go version if modFile is nil. -func modFileGoVersion() string { - _ = TODOWorkspaces("this is obviously wrong.") - // Yes we're picking arbitrarily, we'll have to pass through the version - // we care about - modFile := MainModules.ModFile(MainModules.Versions()[0]) +// in modFile are interpreted, or the latest Go version if modFile is nil. +func modFileGoVersion(modFile *modfile.File) string { if modFile == nil { return LatestGoVersion() }