cmd/covdata: add tools to read/manipulate coverage data files

Add a set of helper packages for reading collections of related
meta-data and counter-data files ("pods") produced by runs of
coverage-instrumented binaries, and a new tool program (cmd/covdata)
for dumping and/or manipulating coverage data files.

Currently "go tool covdata" subcommands include 'merge', 'intersect',
'percent', 'pkglist', 'subtract', 'debugdump', and 'textfmt'
(conversion to the legacy "go tool cover" format).

Updates #51430.

Change-Id: I44167c578f574b4636ab8726e726388531fd3258
Reviewed-on: https://go-review.googlesource.com/c/go/+/357609
Run-TryBot: Than McIntosh <thanm@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: David Chase <drchase@google.com>

src/internal/coverage/cformat/format.go

@@ -0,0 +1,340 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cformat
+
+// This package provides apis for producing human-readable summaries
+// of coverage data (e.g. a coverage percentage for a given package or
+// set of packages) and for writing data in the legacy test format
+// emitted by "go test -coverprofile=<outfile>".
+//
+// The model for using these apis is to create a Formatter object,
+// then make a series of calls to SetPackage and AddUnit passing in
+// data read from coverage meta-data and counter-data files. E.g.
+//
+//		myformatter := cformat.NewFormatter()
+//		...
+//		for each package P in meta-data file: {
+//			myformatter.SetPackage(P)
+//			for each function F in P: {
+//				for each coverable unit U in F: {
+//					myformatter.AddUnit(U)
+//				}
+//			}
+//		}
+//		myformatter.EmitPercent(os.Stdout, "")
+//		myformatter.EmitTextual(somefile)
+//
+// These apis are linked into tests that are built with "-cover", and
+// called at the end of test execution to produce text output or
+// emit coverage percentages.
+
+import (
+	"fmt"
+	"internal/coverage"
+	"internal/coverage/cmerge"
+	"io"
+	"sort"
+	"text/tabwriter"
+)
+
+type Formatter struct {
+	// Maps import path to package state.
+	pm map[string]*pstate
+	// Records current package being visited.
+	pkg string
+	// Pointer to current package state.
+	p *pstate
+	// Counter mode.
+	cm coverage.CounterMode
+}
+
+// pstate records package-level coverage data state:
+// - a table of functions (file/fname/literal)
+// - a map recording the index/ID of each func encountered so far
+// - a table storing execution count for the coverable units in each func
+type pstate struct {
+	// slice of unique functions
+	funcs []fnfile
+	// maps function to index in slice above (index acts as function ID)
+	funcTable map[fnfile]uint32
+
+	// A table storing coverage counts for each coverable unit.
+	unitTable map[extcu]uint32
+}
+
+// extcu encapsulates a coverable unit within some function.
+type extcu struct {
+	fnfid uint32 // index into p.funcs slice
+	coverage.CoverableUnit
+}
+
+// fnfile is a function-name/file-name tuple.
+type fnfile struct {
+	file  string
+	fname string
+	lit   bool
+}
+
+func NewFormatter(cm coverage.CounterMode) *Formatter {
+	return &Formatter{
+		pm: make(map[string]*pstate),
+		cm: cm,
+	}
+}
+
+// SetPackage tells the formatter that we're about to visit the
+// coverage data for the package with the specified import path.
+// Note that it's OK to call SetPackage more than once with the
+// same import path; counter data values will be accumulated.
+func (fm *Formatter) SetPackage(importpath string) {
+	if importpath == fm.pkg {
+		return
+	}
+	fm.pkg = importpath
+	ps, ok := fm.pm[importpath]
+	if !ok {
+		ps = new(pstate)
+		fm.pm[importpath] = ps
+		ps.unitTable = make(map[extcu]uint32)
+		ps.funcTable = make(map[fnfile]uint32)
+	}
+	fm.p = ps
+}
+
+// AddUnit passes info on a single coverable unit (file, funcname,
+// literal flag, range of lines, and counter value) to the formatter.
+// Counter values will be accumulated where appropriate.
+func (fm *Formatter) AddUnit(file string, fname string, isfnlit bool, unit coverage.CoverableUnit, count uint32) {
+	if fm.p == nil {
+		panic("AddUnit invoked before SetPackage")
+	}
+	fkey := fnfile{file: file, fname: fname, lit: isfnlit}
+	idx, ok := fm.p.funcTable[fkey]
+	if !ok {
+		idx = uint32(len(fm.p.funcs))
+		fm.p.funcs = append(fm.p.funcs, fkey)
+		fm.p.funcTable[fkey] = idx
+	}
+	ukey := extcu{fnfid: idx, CoverableUnit: unit}
+	pcount := fm.p.unitTable[ukey]
+	var result uint32
+	if fm.cm == coverage.CtrModeSet {
+		if count != 0 || pcount != 0 {
+			result = 1
+		}
+	} else {
+		// Use saturating arithmetic.
+		result, _ = cmerge.SaturatingAdd(pcount, count)
+	}
+	fm.p.unitTable[ukey] = result
+}
+
+// sortUnits sorts a slice of extcu objects in a package according to
+// source position information (e.g. file and line). Note that we don't
+// include function name as part of the sorting criteria, the thinking
+// being that is better to provide things in the original source order.
+func (p *pstate) sortUnits(units []extcu) {
+	sort.Slice(units, func(i, j int) bool {
+		ui := units[i]
+		uj := units[j]
+		ifile := p.funcs[ui.fnfid].file
+		jfile := p.funcs[uj.fnfid].file
+		if ifile != jfile {
+			return ifile < jfile
+		}
+		// NB: not taking function literal flag into account here (no
+		// need, since other fields are guaranteed to be distinct).
+		if units[i].StLine != units[j].StLine {
+			return units[i].StLine < units[j].StLine
+		}
+		if units[i].EnLine != units[j].EnLine {
+			return units[i].EnLine < units[j].EnLine
+		}
+		if units[i].StCol != units[j].StCol {
+			return units[i].StCol < units[j].StCol
+		}
+		if units[i].EnCol != units[j].EnCol {
+			return units[i].EnCol < units[j].EnCol
+		}
+		return units[i].NxStmts < units[j].NxStmts
+	})
+}
+
+// EmitTextual writes the accumulated coverage data in the legacy
+// cmd/cover text format to the writer 'w'. We sort the data items by
+// importpath, source file, and line number before emitting (this sorting
+// is not explicitly mandated by the format, but seems like a good idea
+// for repeatable/deterministic dumps).
+func (fm *Formatter) EmitTextual(w io.Writer) error {
+	if fm.cm == coverage.CtrModeInvalid {
+		panic("internal error, counter mode unset")
+	}
+	if _, err := fmt.Fprintf(w, "mode: %s\n", fm.cm.String()); err != nil {
+		return err
+	}
+	pkgs := make([]string, 0, len(fm.pm))
+	for importpath := range fm.pm {
+		pkgs = append(pkgs, importpath)
+	}
+	sort.Strings(pkgs)
+	for _, importpath := range pkgs {
+		p := fm.pm[importpath]
+		units := make([]extcu, 0, len(p.unitTable))
+		for u := range p.unitTable {
+			units = append(units, u)
+		}
+		p.sortUnits(units)
+		for _, u := range units {
+			count := p.unitTable[u]
+			file := p.funcs[u.fnfid].file
+			if _, err := fmt.Fprintf(w, "%s:%d.%d,%d.%d %d %d\n",
+				file, u.StLine, u.StCol,
+				u.EnLine, u.EnCol, u.NxStmts, count); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// EmitPercent writes out a "percentage covered" string to the writer 'w'.
+func (fm *Formatter) EmitPercent(w io.Writer, covpkgs string, noteEmpty bool) error {
+	pkgs := make([]string, 0, len(fm.pm))
+	for importpath := range fm.pm {
+		pkgs = append(pkgs, importpath)
+	}
+	sort.Strings(pkgs)
+	seenPkg := false
+	for _, importpath := range pkgs {
+		seenPkg = true
+		p := fm.pm[importpath]
+		var totalStmts, coveredStmts uint64
+		for unit, count := range p.unitTable {
+			nx := uint64(unit.NxStmts)
+			totalStmts += nx
+			if count != 0 {
+				coveredStmts += nx
+			}
+		}
+		if _, err := fmt.Fprintf(w, "\t%s\t", importpath); err != nil {
+			return err
+		}
+		if totalStmts == 0 {
+			if _, err := fmt.Fprintf(w, "coverage: [no statements]\n"); err != nil {
+				return err
+			}
+		} else {
+			if _, err := fmt.Fprintf(w, "coverage: %.1f%% of statements%s\n", 100*float64(coveredStmts)/float64(totalStmts), covpkgs); err != nil {
+				return err
+			}
+		}
+	}
+	if noteEmpty && !seenPkg {
+		if _, err := fmt.Fprintf(w, "coverage: [no statements]\n"); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// EmitFuncs writes out a function-level summary to the writer 'w'. A
+// note on handling function literals: although we collect coverage
+// data for unnamed literals, it probably does not make sense to
+// include them in the function summary since there isn't any good way
+// to name them (this is also consistent with the legacy cmd/cover
+// implementation). We do want to include their counts in the overall
+// summary however.
+func (fm *Formatter) EmitFuncs(w io.Writer) error {
+	if fm.cm == coverage.CtrModeInvalid {
+		panic("internal error, counter mode unset")
+	}
+	perc := func(covered, total uint64) float64 {
+		if total == 0 {
+			total = 1
+		}
+		return 100.0 * float64(covered) / float64(total)
+	}
+	tabber := tabwriter.NewWriter(w, 1, 8, 1, '\t', 0)
+	defer tabber.Flush()
+	allStmts := uint64(0)
+	covStmts := uint64(0)
+
+	pkgs := make([]string, 0, len(fm.pm))
+	for importpath := range fm.pm {
+		pkgs = append(pkgs, importpath)
+	}
+	sort.Strings(pkgs)
+
+	// Emit functions for each package, sorted by import path.
+	for _, importpath := range pkgs {
+		p := fm.pm[importpath]
+		if len(p.unitTable) == 0 {
+			continue
+		}
+		units := make([]extcu, 0, len(p.unitTable))
+		for u := range p.unitTable {
+			units = append(units, u)
+		}
+
+		// Within a package, sort the units, then walk through the
+		// sorted array. Each time we hit a new function, emit the
+		// summary entry for the previous function, then make one last
+		// emit call at the end of the loop.
+		p.sortUnits(units)
+		fname := ""
+		ffile := ""
+		flit := false
+		var fline uint32
+		var cstmts, tstmts uint64
+		captureFuncStart := func(u extcu) {
+			fname = p.funcs[u.fnfid].fname
+			ffile = p.funcs[u.fnfid].file
+			flit = p.funcs[u.fnfid].lit
+			fline = u.StLine
+		}
+		emitFunc := func(u extcu) error {
+			// Don't emit entries for function literals (see discussion
+			// in function header comment above).
+			if !flit {
+				if _, err := fmt.Fprintf(tabber, "%s:%d:\t%s\t%.1f%%\n",
+					ffile, fline, fname, perc(cstmts, tstmts)); err != nil {
+					return err
+				}
+			}
+			captureFuncStart(u)
+			allStmts += tstmts
+			covStmts += cstmts
+			tstmts = 0
+			cstmts = 0
+			return nil
+		}
+		for k, u := range units {
+			if k == 0 {
+				captureFuncStart(u)
+			} else {
+				if fname != p.funcs[u.fnfid].fname {
+					// New function; emit entry for previous one.
+					if err := emitFunc(u); err != nil {
+						return err
+					}
+				}
+			}
+			tstmts += uint64(u.NxStmts)
+			count := p.unitTable[u]
+			if count != 0 {
+				cstmts += uint64(u.NxStmts)
+			}
+		}
+		if err := emitFunc(extcu{}); err != nil {
+			return err
+		}
+	}
+	if _, err := fmt.Fprintf(tabber, "%s\t%s\t%.1f%%\n",
+		"total", "(statements)", perc(covStmts, allStmts)); err != nil {
+		return err
+	}
+	return nil
+}