runtime/coverage: add support for "auxiliary" meta-data files

Enhance the functions called by _testmain.go during "go test -cover"
test binary runs to allow for injection of extra or "auxiliary"
meta-data files when reporting coverage statistics. There are unit
tests for this functionality, but it is not yet wired up to be used by
the Go command yet, that will appear in a subsequent patch.

Change-Id: I10b79ca003fd7a875727dc1a86f23f58d6bf630c
Reviewed-on: https://go-review.googlesource.com/c/go/+/495451
Run-TryBot: Than McIntosh <thanm@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: David Chase <drchase@google.com>

src/go/build/deps_test.go

@@ -621,6 +621,7 @@ var depsRules = `
 	internal/coverage/cmerge
 	< internal/coverage/cformat;
 
+    encoding/json,
 	runtime/debug,
 	internal/coverage/calloc,
 	internal/coverage/cformat,

src/internal/coverage/cmddefs.go

@@ -71,3 +71,17 @@ type CoverFixupConfig struct {
 	// Counter granularity (perblock or perfunc).
 	CounterGranularity string
 }
+
+// MetaFilePaths contains information generated by the Go command and
+// the read in by coverage test support functions within an executing
+// "go test -cover" binary.
+type MetaFileCollection struct {
+	ImportPaths       []string
+	MetaFileFragments []string
+}
+
+// Name of file within the "go test -cover" temp coverdir directory
+// containing a list of meta-data files for packages being tested
+// in a "go test -coverpkg=... ..." run. This constant is shared
+// by the Go command and by the coverage runtime.
+const MetaFilesFileName = "metafiles.txt"

src/runtime/coverage/testsupport.go

@@ -5,6 +5,7 @@
 package coverage
 
 import (
+	"encoding/json"
 	"fmt"
 	"internal/coverage"
 	"internal/coverage/calloc"
@@ -15,6 +16,7 @@ import (
 	"internal/coverage/pods"
 	"io"
 	"os"
+	"path/filepath"
 	"runtime/internal/atomic"
 	"strings"
 	"unsafe"
@@ -88,11 +90,19 @@ func processCoverTestDirInternal(dir string, cfile string, cm string, cpkg strin
 	// hash (just in case there are multiple instrumented executables
 	// in play). See issue #57924 for more on this.
 	hashstring := fmt.Sprintf("%x", finalHash)
+	importpaths := make(map[string]struct{})
 	for _, p := range podlist {
 		if !strings.Contains(p.MetaFile, hashstring) {
 			continue
 		}
-		if err := ts.processPod(p); err != nil {
+		if err := ts.processPod(p, importpaths); err != nil {
+			return err
+		}
+	}
+
+	metafilespath := filepath.Join(dir, coverage.MetaFilesFileName)
+	if _, err := os.Stat(metafilespath); err == nil {
+		if err := ts.readAuxMetaFiles(metafilespath, importpaths); err != nil {
 			return err
 		}
 	}
@@ -124,7 +134,7 @@ type tstate struct {
 }
 
 // processPod reads coverage counter data for a specific pod.
-func (ts *tstate) processPod(p pods.Pod) error {
+func (ts *tstate) processPod(p pods.Pod, importpaths map[string]struct{}) error {
 	// Open meta-data file
 	f, err := os.Open(p.MetaFile)
 	if err != nil {
@@ -204,6 +214,7 @@ func (ts *tstate) processPod(p pods.Pod) error {
 			return fmt.Errorf("reading pkg %d from meta-file %s: %s", pkIdx, p.MetaFile, err)
 		}
 		ts.cf.SetPackage(pd.PackagePath())
+		importpaths[pd.PackagePath()] = struct{}{}
 		var fd coverage.FuncDesc
 		nf := pd.NumFuncs()
 		for fnIdx := uint32(0); fnIdx < nf; fnIdx++ {
@@ -235,6 +246,37 @@ type pkfunc struct {
 	pk, fcn uint32
 }
 
+func (ts *tstate) readAuxMetaFiles(metafiles string, importpaths map[string]struct{}) error {
+	// Unmarshall the information on available aux metafiles into
+	// a MetaFileCollection struct.
+	var mfc coverage.MetaFileCollection
+	data, err := os.ReadFile(metafiles)
+	if err != nil {
+		return fmt.Errorf("error reading auxmetafiles file %q: %v", metafiles, err)
+	}
+	if err := json.Unmarshal(data, &mfc); err != nil {
+		return fmt.Errorf("error reading auxmetafiles file %q: %v", metafiles, err)
+	}
+
+	// Walk through each available aux meta-file. If we've already
+	// seen the package path in question during the walk of the
+	// "regular" meta-data file, then we can skip the package,
+	// otherwise construct a dummy pod with the single meta-data file
+	// (no counters) and invoke processPod on it.
+	for i := range mfc.ImportPaths {
+		p := mfc.ImportPaths[i]
+		if _, ok := importpaths[p]; ok {
+			continue
+		}
+		var pod pods.Pod
+		pod.MetaFile = mfc.MetaFileFragments[i]
+		if err := ts.processPod(pod, importpaths); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // snapshot returns a snapshot of coverage percentage at a moment of
 // time within a running test, so as to support the testing.Coverage()
 // function. This version doesn't examine coverage meta-data, so the

src/runtime/coverage/ts_test.go

@@ -5,8 +5,12 @@
 package coverage
 
 import (
+	"encoding/json"
+	"internal/coverage"
 	"internal/goexperiment"
+	"internal/testenv"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -16,6 +20,14 @@ import (
 //go:linkname testing_testGoCoverDir testing.testGoCoverDir
 func testing_testGoCoverDir() string
 
+func testGoCoverDir(t *testing.T) string {
+	tgcd := testing_testGoCoverDir()
+	if tgcd != "" {
+		return tgcd
+	}
+	return t.TempDir()
+}
+
 // TestTestSupport does a basic verification of the functionality in
 // runtime/coverage.processCoverTestDir (doing this here as opposed to
 // relying on other test paths will provide a better signal when
@@ -27,12 +39,13 @@ func TestTestSupport(t *testing.T) {
 	if testing.CoverMode() == "" {
 		return
 	}
+	tgcd := testGoCoverDir(t)
 	t.Logf("testing.testGoCoverDir() returns %s mode=%s\n",
-		testing_testGoCoverDir(), testing.CoverMode())
+		tgcd, testing.CoverMode())
 
 	textfile := filepath.Join(t.TempDir(), "file.txt")
 	var sb strings.Builder
-	err := processCoverTestDirInternal(testing_testGoCoverDir(), textfile,
+	err := processCoverTestDirInternal(tgcd, textfile,
 		testing.CoverMode(), "", &sb)
 	if err != nil {
 		t.Fatalf("bad: %v", err)
@@ -93,3 +106,102 @@ func TestCoverageSnapshot(t *testing.T) {
 			cond, C1, C2)
 	}
 }
+
+const hellogo = `
+package main
+
+func main() {
+  println("hello")
+}
+`
+
+// Returns a pair F,T where F is a meta-data file generated from
+// "hello.go" above, and T is a token to look for that should be
+// present in the coverage report from F.
+func genAuxMeta(t *testing.T, dstdir string) (string, string) {
+	// Do a GOCOVERDIR=<tmp> go run hello.go
+	src := filepath.Join(dstdir, "hello.go")
+	if err := os.WriteFile(src, []byte(hellogo), 0777); err != nil {
+		t.Fatalf("write failed: %v", err)
+	}
+	args := []string{"run", "-covermode=" + testing.CoverMode(), src}
+	cmd := exec.Command(testenv.GoToolPath(t), args...)
+	cmd.Env = updateGoCoverDir(os.Environ(), dstdir, true)
+	if b, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("go run failed (%v): %s", err, b)
+	}
+
+	// Pick out the generated meta-data file.
+	files, err := os.ReadDir(dstdir)
+	if err != nil {
+		t.Fatalf("reading %s: %v", dstdir, err)
+	}
+	for _, f := range files {
+		if strings.HasPrefix(f.Name(), "covmeta") {
+			return filepath.Join(dstdir, f.Name()), "hello.go:"
+		}
+	}
+	t.Fatalf("could not locate generated meta-data file")
+	return "", ""
+}
+
+func TestAuxMetaDataFiles(t *testing.T) {
+	if !goexperiment.CoverageRedesign {
+		return
+	}
+	if testing.CoverMode() == "" {
+		return
+	}
+	testenv.MustHaveGoRun(t)
+	tgcd := testGoCoverDir(t)
+	t.Logf("testing.testGoCoverDir() returns %s mode=%s\n",
+		tgcd, testing.CoverMode())
+
+	td := t.TempDir()
+
+	// Manufacture a new, separate meta-data file not related to this
+	// test. Contents are not important, just so long as the
+	// packages/paths are different.
+	othermetadir := filepath.Join(td, "othermeta")
+	if err := os.Mkdir(othermetadir, 0777); err != nil {
+		t.Fatalf("mkdir failed: %v", err)
+	}
+	mfile, token := genAuxMeta(t, othermetadir)
+
+	// Write a metafiles file.
+	metafiles := filepath.Join(tgcd, coverage.MetaFilesFileName)
+	mfc := coverage.MetaFileCollection{
+		ImportPaths:       []string{"command-line-arguments"},
+		MetaFileFragments: []string{mfile},
+	}
+	jdata, err := json.Marshal(mfc)
+	if err != nil {
+		t.Fatalf("marshal MetaFileCollection: %v", err)
+	}
+	if err := os.WriteFile(metafiles, jdata, 0666); err != nil {
+		t.Fatalf("write failed: %v", err)
+	}
+
+	// Kick off guts of test.
+	var sb strings.Builder
+	textfile := filepath.Join(td, "file2.txt")
+	err = processCoverTestDirInternal(tgcd, textfile,
+		testing.CoverMode(), "", &sb)
+	if err != nil {
+		t.Fatalf("bad: %v", err)
+	}
+	if err = os.Remove(metafiles); err != nil {
+		t.Fatalf("removing metafiles file: %v", err)
+	}
+
+	// Look for the expected things in the coverage profile.
+	contents, err := os.ReadFile(textfile)
+	strc := string(contents)
+	if err != nil {
+		t.Fatalf("problems reading text file %s: %v", textfile, err)
+	}
+	if !strings.Contains(strc, token) {
+		t.Logf("content: %s\n", string(contents))
+		t.Fatalf("cov profile does not contain aux meta content %q", token)
+	}
+}