metamerge.go

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  // This file contains functions and apis that support merging of
     8  // meta-data information.  It helps implement the "merge", "subtract",
     9  // and "intersect" subcommands.
    10  
    11  import (
    12  	"fmt"
    13  	"hash/fnv"
    14  	"internal/coverage"
    15  	"internal/coverage/calloc"
    16  	"internal/coverage/cmerge"
    17  	"internal/coverage/decodecounter"
    18  	"internal/coverage/decodemeta"
    19  	"internal/coverage/encodecounter"
    20  	"internal/coverage/encodemeta"
    21  	"internal/coverage/slicewriter"
    22  	"io"
    23  	"os"
    24  	"path/filepath"
    25  	"sort"
    26  	"time"
    27  	"unsafe"
    28  )
    29  
    30  // metaMerge provides state and methods to help manage the process
    31  // of selecting or merging meta data files. There are three cases
    32  // of interest here: the "-pcombine" flag provided by merge, the
    33  // "-pkg" option provided by all merge/subtract/intersect, and
    34  // a regular vanilla merge with no package selection
    35  //
    36  // In the -pcombine case, we're essentially glomming together all the
    37  // meta-data for all packages and all functions, meaning that
    38  // everything we see in a given package needs to be added into the
    39  // meta-data file builder; we emit a single meta-data file at the end
    40  // of the run.
    41  //
    42  // In the -pkg case, we will typically emit a single meta-data file
    43  // per input pod, where that new meta-data file contains entries for
    44  // just the selected packages.
    45  //
    46  // In the third case (vanilla merge with no combining or package
    47  // selection) we can carry over meta-data files without touching them
    48  // at all (only counter data files will be merged).
    49  type metaMerge struct {
    50  	calloc.BatchCounterAlloc
    51  	cmerge.Merger
    52  	// maps package import path to package state
    53  	pkm map[string]*pkstate
    54  	// list of packages
    55  	pkgs []*pkstate
    56  	// current package state
    57  	p *pkstate
    58  	// current pod state
    59  	pod *podstate
    60  	// counter data file osargs/goos/goarch state
    61  	astate *argstate
    62  }
    63  
    64  // pkstate
    65  type pkstate struct {
    66  	// index of package within meta-data file.
    67  	pkgIdx uint32
    68  	// this maps function index within the package to counter data payload
    69  	ctab map[uint32]decodecounter.FuncPayload
    70  	// pointer to meta-data blob for package
    71  	mdblob []byte
    72  	// filled in only for -pcombine merges
    73  	*pcombinestate
    74  }
    75  
    76  type podstate struct {
    77  	pmm      map[pkfunc]decodecounter.FuncPayload
    78  	mdf      string
    79  	mfr      *decodemeta.CoverageMetaFileReader
    80  	fileHash [16]byte
    81  }
    82  
    83  type pkfunc struct {
    84  	pk, fcn uint32
    85  }
    86  
    87  // pcombinestate
    88  type pcombinestate struct {
    89  	// Meta-data builder for the package.
    90  	cmdb *encodemeta.CoverageMetaDataBuilder
    91  	// Maps function meta-data hash to new function index in the
    92  	// new version of the package we're building.
    93  	ftab map[[16]byte]uint32
    94  }
    95  
    96  func newMetaMerge() *metaMerge {
    97  	return &metaMerge{
    98  		pkm:    make(map[string]*pkstate),
    99  		astate: &argstate{},
   100  	}
   101  }
   102  
   103  func (mm *metaMerge) visitMetaDataFile(mdf string, mfr *decodemeta.CoverageMetaFileReader) {
   104  	dbgtrace(2, "visitMetaDataFile(mdf=%s)", mdf)
   105  
   106  	// Record meta-data file name.
   107  	mm.pod.mdf = mdf
   108  	// Keep a pointer to the file-level reader.
   109  	mm.pod.mfr = mfr
   110  	// Record file hash.
   111  	mm.pod.fileHash = mfr.FileHash()
   112  	// Counter mode and granularity -- detect and record clashes here.
   113  	newgran := mfr.CounterGranularity()
   114  	newmode := mfr.CounterMode()
   115  	if err := mm.SetModeAndGranularity(mdf, newmode, newgran); err != nil {
   116  		fatal("%v", err)
   117  	}
   118  }
   119  
   120  func (mm *metaMerge) beginCounterDataFile(cdr *decodecounter.CounterDataReader) {
   121  	state := argvalues{
   122  		osargs: cdr.OsArgs(),
   123  		goos:   cdr.Goos(),
   124  		goarch: cdr.Goarch(),
   125  	}
   126  	mm.astate.Merge(state)
   127  }
   128  
   129  func copyMetaDataFile(inpath, outpath string) {
   130  	inf, err := os.Open(inpath)
   131  	if err != nil {
   132  		fatal("opening input meta-data file %s: %v", inpath, err)
   133  	}
   134  	defer inf.Close()
   135  
   136  	fi, err := inf.Stat()
   137  	if err != nil {
   138  		fatal("accessing input meta-data file %s: %v", inpath, err)
   139  	}
   140  
   141  	outf, err := os.OpenFile(outpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fi.Mode())
   142  	if err != nil {
   143  		fatal("opening output meta-data file %s: %v", outpath, err)
   144  	}
   145  
   146  	_, err = io.Copy(outf, inf)
   147  	outf.Close()
   148  	if err != nil {
   149  		fatal("writing output meta-data file %s: %v", outpath, err)
   150  	}
   151  }
   152  
   153  func (mm *metaMerge) beginPod() {
   154  	mm.pod = &podstate{
   155  		pmm: make(map[pkfunc]decodecounter.FuncPayload),
   156  	}
   157  }
   158  
   159  // metaEndPod handles actions needed when we're done visiting all of
   160  // the things in a pod -- counter files and meta-data file. There are
   161  // three cases of interest here:
   162  //
   163  // Case 1: in an unconditional merge (we're not selecting a specific set of
   164  // packages using "-pkg", and the "-pcombine" option is not in use),
   165  // we can simply copy over the meta-data file from input to output.
   166  //
   167  // Case 2: if this is a select merge (-pkg is in effect), then at
   168  // this point we write out a new smaller meta-data file that includes
   169  // only the packages of interest. At this point we also emit a merged
   170  // counter data file as well.
   171  //
   172  // Case 3: if "-pcombine" is in effect, we don't write anything at
   173  // this point (all writes will happen at the end of the run).
   174  func (mm *metaMerge) endPod(pcombine bool) {
   175  	if pcombine {
   176  		// Just clear out the pod data, we'll do all the
   177  		// heavy lifting at the end.
   178  		mm.pod = nil
   179  		return
   180  	}
   181  
   182  	finalHash := mm.pod.fileHash
   183  	if matchpkg != nil {
   184  		// Emit modified meta-data file for this pod.
   185  		finalHash = mm.emitMeta(*outdirflag, pcombine)
   186  	} else {
   187  		// Copy meta-data file for this pod to the output directory.
   188  		inpath := mm.pod.mdf
   189  		mdfbase := filepath.Base(mm.pod.mdf)
   190  		outpath := filepath.Join(*outdirflag, mdfbase)
   191  		copyMetaDataFile(inpath, outpath)
   192  	}
   193  
   194  	// Emit accumulated counter data for this pod.
   195  	mm.emitCounters(*outdirflag, finalHash)
   196  
   197  	// Reset package state.
   198  	mm.pkm = make(map[string]*pkstate)
   199  	mm.pkgs = nil
   200  	mm.pod = nil
   201  
   202  	// Reset counter mode and granularity
   203  	mm.ResetModeAndGranularity()
   204  }
   205  
   206  // emitMeta encodes and writes out a new coverage meta-data file as
   207  // part of a merge operation, specifically a merge with the
   208  // "-pcombine" flag.
   209  func (mm *metaMerge) emitMeta(outdir string, pcombine bool) [16]byte {
   210  	fh := fnv.New128a()
   211  	fhSum := fnv.New128a()
   212  	blobs := [][]byte{}
   213  	tlen := uint64(unsafe.Sizeof(coverage.MetaFileHeader{}))
   214  	for _, p := range mm.pkgs {
   215  		var blob []byte
   216  		if pcombine {
   217  			mdw := &slicewriter.WriteSeeker{}
   218  			p.cmdb.Emit(mdw)
   219  			blob = mdw.BytesWritten()
   220  		} else {
   221  			blob = p.mdblob
   222  		}
   223  		fhSum.Reset()
   224  		fhSum.Write(blob)
   225  		ph := fhSum.Sum(nil)
   226  		blobs = append(blobs, blob)
   227  		if _, err := fh.Write(ph[:]); err != nil {
   228  			panic(fmt.Sprintf("internal error: md5 sum failed: %v", err))
   229  		}
   230  		tlen += uint64(len(blob))
   231  	}
   232  	var finalHash [16]byte
   233  	fhh := fh.Sum(nil)
   234  	copy(finalHash[:], fhh)
   235  
   236  	// Open meta-file for writing.
   237  	fn := fmt.Sprintf("%s.%x", coverage.MetaFilePref, finalHash)
   238  	fpath := filepath.Join(outdir, fn)
   239  	mf, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
   240  	if err != nil {
   241  		fatal("unable to open output meta-data file %s: %v", fpath, err)
   242  	}
   243  
   244  	// Encode and write.
   245  	mfw := encodemeta.NewCoverageMetaFileWriter(fpath, mf)
   246  	err = mfw.Write(finalHash, blobs, mm.Mode(), mm.Granularity())
   247  	if err != nil {
   248  		fatal("error writing %s: %v\n", fpath, err)
   249  	}
   250  	return finalHash
   251  }
   252  
   253  func (mm *metaMerge) emitCounters(outdir string, metaHash [16]byte) {
   254  	// Open output file. The file naming scheme is intended to mimic
   255  	// that used when running a coverage-instrumented binary, for
   256  	// consistency (however the process ID is not meaningful here, so
   257  	// use a value of zero).
   258  	var dummyPID int
   259  	fn := fmt.Sprintf(coverage.CounterFileTempl, coverage.CounterFilePref, metaHash, dummyPID, time.Now().UnixNano())
   260  	fpath := filepath.Join(outdir, fn)
   261  	cf, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
   262  	if err != nil {
   263  		fatal("opening counter data file %s: %v", fpath, err)
   264  	}
   265  	defer func() {
   266  		if err := cf.Close(); err != nil {
   267  			fatal("error closing output meta-data file %s: %v", fpath, err)
   268  		}
   269  	}()
   270  
   271  	args := mm.astate.ArgsSummary()
   272  	cfw := encodecounter.NewCoverageDataWriter(cf, coverage.CtrULeb128)
   273  	if err := cfw.Write(metaHash, args, mm); err != nil {
   274  		fatal("counter file write failed: %v", err)
   275  	}
   276  	mm.astate = &argstate{}
   277  }
   278  
   279  // VisitFuncs is used while writing the counter data files; it
   280  // implements the 'VisitFuncs' method required by the interface
   281  // internal/coverage/encodecounter/CounterVisitor.
   282  func (mm *metaMerge) VisitFuncs(f encodecounter.CounterVisitorFn) error {
   283  	if *verbflag >= 4 {
   284  		fmt.Printf("counterVisitor invoked\n")
   285  	}
   286  	// For each package, for each function, construct counter
   287  	// array and then call "f" on it.
   288  	for pidx, p := range mm.pkgs {
   289  		fids := make([]int, 0, len(p.ctab))
   290  		for fid := range p.ctab {
   291  			fids = append(fids, int(fid))
   292  		}
   293  		sort.Ints(fids)
   294  		if *verbflag >= 4 {
   295  			fmt.Printf("fids for pk=%d: %+v\n", pidx, fids)
   296  		}
   297  		for _, fid := range fids {
   298  			fp := p.ctab[uint32(fid)]
   299  			if *verbflag >= 4 {
   300  				fmt.Printf("counter write for pk=%d fid=%d len(ctrs)=%d\n", pidx, fid, len(fp.Counters))
   301  			}
   302  			if err := f(uint32(pidx), uint32(fid), fp.Counters); err != nil {
   303  				return err
   304  			}
   305  		}
   306  	}
   307  	return nil
   308  }
   309  
   310  func (mm *metaMerge) visitPackage(pd *decodemeta.CoverageMetaDataDecoder, pkgIdx uint32, pcombine bool) {
   311  	p, ok := mm.pkm[pd.PackagePath()]
   312  	if !ok {
   313  		p = &pkstate{
   314  			pkgIdx: uint32(len(mm.pkgs)),
   315  		}
   316  		mm.pkgs = append(mm.pkgs, p)
   317  		mm.pkm[pd.PackagePath()] = p
   318  		if pcombine {
   319  			p.pcombinestate = new(pcombinestate)
   320  			cmdb, err := encodemeta.NewCoverageMetaDataBuilder(pd.PackagePath(), pd.PackageName(), pd.ModulePath())
   321  			if err != nil {
   322  				fatal("fatal error creating meta-data builder: %v", err)
   323  			}
   324  			dbgtrace(2, "install new pkm entry for package %s pk=%d", pd.PackagePath(), pkgIdx)
   325  			p.cmdb = cmdb
   326  			p.ftab = make(map[[16]byte]uint32)
   327  		} else {
   328  			var err error
   329  			p.mdblob, err = mm.pod.mfr.GetPackagePayload(pkgIdx, nil)
   330  			if err != nil {
   331  				fatal("error extracting package %d payload from %s: %v",
   332  					pkgIdx, mm.pod.mdf, err)
   333  			}
   334  		}
   335  		p.ctab = make(map[uint32]decodecounter.FuncPayload)
   336  	}
   337  	mm.p = p
   338  }
   339  
   340  func (mm *metaMerge) visitFuncCounterData(data decodecounter.FuncPayload) {
   341  	key := pkfunc{pk: data.PkgIdx, fcn: data.FuncIdx}
   342  	val := mm.pod.pmm[key]
   343  	// FIXME: in theory either A) len(val.Counters) is zero, or B)
   344  	// the two lengths are equal. Assert if not? Of course, we could
   345  	// see odd stuff if there is source file skew.
   346  	if *verbflag > 4 {
   347  		fmt.Printf("visit pk=%d fid=%d len(counters)=%d\n", data.PkgIdx, data.FuncIdx, len(data.Counters))
   348  	}
   349  	if len(val.Counters) < len(data.Counters) {
   350  		t := val.Counters
   351  		val.Counters = mm.AllocateCounters(len(data.Counters))
   352  		copy(val.Counters, t)
   353  	}
   354  	err, overflow := mm.MergeCounters(val.Counters, data.Counters)
   355  	if err != nil {
   356  		fatal("%v", err)
   357  	}
   358  	if overflow {
   359  		warn("uint32 overflow during counter merge")
   360  	}
   361  	mm.pod.pmm[key] = val
   362  }
   363  
   364  func (mm *metaMerge) visitFunc(pkgIdx uint32, fnIdx uint32, fd *coverage.FuncDesc, verb string, pcombine bool) {
   365  	if *verbflag >= 3 {
   366  		fmt.Printf("visit pk=%d fid=%d func %s\n", pkgIdx, fnIdx, fd.Funcname)
   367  	}
   368  
   369  	var counters []uint32
   370  	key := pkfunc{pk: pkgIdx, fcn: fnIdx}
   371  	v, haveCounters := mm.pod.pmm[key]
   372  	if haveCounters {
   373  		counters = v.Counters
   374  	}
   375  
   376  	if pcombine {
   377  		// If the merge is running in "combine programs" mode, then hash
   378  		// the function and look it up in the package ftab to see if we've
   379  		// encountered it before. If we haven't, then register it with the
   380  		// meta-data builder.
   381  		fnhash := encodemeta.HashFuncDesc(fd)
   382  		gfidx, ok := mm.p.ftab[fnhash]
   383  		if !ok {
   384  			// We haven't seen this function before, need to add it to
   385  			// the meta data.
   386  			gfidx = uint32(mm.p.cmdb.AddFunc(*fd))
   387  			mm.p.ftab[fnhash] = gfidx
   388  			if *verbflag >= 3 {
   389  				fmt.Printf("new meta entry for fn %s fid=%d\n", fd.Funcname, gfidx)
   390  			}
   391  		}
   392  		fnIdx = gfidx
   393  	}
   394  	if !haveCounters {
   395  		return
   396  	}
   397  
   398  	// Install counters in package ctab.
   399  	gfp, ok := mm.p.ctab[fnIdx]
   400  	if ok {
   401  		if verb == "subtract" || verb == "intersect" {
   402  			panic("should never see this for intersect/subtract")
   403  		}
   404  		if *verbflag >= 3 {
   405  			fmt.Printf("counter merge for %s fidx=%d\n", fd.Funcname, fnIdx)
   406  		}
   407  		// Merge.
   408  		err, overflow := mm.MergeCounters(gfp.Counters, counters)
   409  		if err != nil {
   410  			fatal("%v", err)
   411  		}
   412  		if overflow {
   413  			warn("uint32 overflow during counter merge")
   414  		}
   415  		mm.p.ctab[fnIdx] = gfp
   416  	} else {
   417  		if *verbflag >= 3 {
   418  			fmt.Printf("null merge for %s fidx %d\n", fd.Funcname, fnIdx)
   419  		}
   420  		gfp := v
   421  		gfp.PkgIdx = mm.p.pkgIdx
   422  		gfp.FuncIdx = fnIdx
   423  		mm.p.ctab[fnIdx] = gfp
   424  	}
   425  }
   426
View as plain text