reports.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package upload
     6  
     7  import (
     8  	"crypto/rand"
     9  	"encoding/binary"
    10  	"encoding/json"
    11  	"fmt"
    12  	"math"
    13  	"os"
    14  	"path/filepath"
    15  	"strings"
    16  	"time"
    17  
    18  	"golang.org/x/telemetry/internal/config"
    19  	"golang.org/x/telemetry/internal/counter"
    20  	"golang.org/x/telemetry/internal/telemetry"
    21  )
    22  
    23  // reports generates reports from inactive count files
    24  func (u *uploader) reports(todo *work) ([]string, error) {
    25  	if mode, _ := u.dir.Mode(); mode == "off" {
    26  		return nil, nil // no reports
    27  	}
    28  	thisInstant := u.startTime
    29  	today := thisInstant.Format(time.DateOnly)
    30  	lastWeek := latestReport(todo.uploaded)
    31  	if lastWeek >= today { //should never happen
    32  		lastWeek = ""
    33  	}
    34  	u.logger.Printf("Last week: %s, today: %s", lastWeek, today)
    35  	countFiles := make(map[string][]string) // expiry date string->filenames
    36  	earliest := make(map[string]time.Time)  // earliest begin time for any counter
    37  	for _, f := range todo.countfiles {
    38  		begin, end, err := u.counterDateSpan(f)
    39  		if err != nil {
    40  			// This shouldn't happen: we should have already skipped count files that
    41  			// don't contain valid start or end times.
    42  			u.logger.Printf("BUG: failed to parse expiry for collected count file: %v", err)
    43  			continue
    44  		}
    45  
    46  		if end.Before(thisInstant) {
    47  			expiry := end.Format(dateFormat)
    48  			countFiles[expiry] = append(countFiles[expiry], f)
    49  			if earliest[expiry].IsZero() || earliest[expiry].After(begin) {
    50  				earliest[expiry] = begin
    51  			}
    52  		}
    53  	}
    54  	for expiry, files := range countFiles {
    55  		if notNeeded(expiry, *todo) {
    56  			u.logger.Printf("Files for %s not needed, deleting %v", expiry, files)
    57  			// The report already exists.
    58  			// There's another check in createReport.
    59  			u.deleteFiles(files)
    60  			continue
    61  		}
    62  		fname, err := u.createReport(earliest[expiry], expiry, files, lastWeek)
    63  		if err != nil {
    64  			u.logger.Printf("Failed to create report for %s: %v", expiry, err)
    65  			continue
    66  		}
    67  		if fname != "" {
    68  			u.logger.Printf("Ready to upload: %s", filepath.Base(fname))
    69  			todo.readyfiles = append(todo.readyfiles, fname)
    70  		}
    71  	}
    72  	return todo.readyfiles, nil
    73  }
    74  
    75  // latestReport returns the YYYY-MM-DD of the last report uploaded
    76  // or the empty string if there are no reports.
    77  func latestReport(uploaded map[string]bool) string {
    78  	var latest string
    79  	for name := range uploaded {
    80  		if strings.HasSuffix(name, ".json") {
    81  			if name > latest {
    82  				latest = name
    83  			}
    84  		}
    85  	}
    86  	if latest == "" {
    87  		return ""
    88  	}
    89  	// strip off the .json
    90  	return latest[:len(latest)-len(".json")]
    91  }
    92  
    93  // notNeeded returns true if the report for date has already been created
    94  func notNeeded(date string, todo work) bool {
    95  	if todo.uploaded != nil && todo.uploaded[date+".json"] {
    96  		return true
    97  	}
    98  	// maybe the report is already in todo.readyfiles
    99  	for _, f := range todo.readyfiles {
   100  		if strings.Contains(f, date) {
   101  			return true
   102  		}
   103  	}
   104  	return false
   105  }
   106  
   107  func (u *uploader) deleteFiles(files []string) {
   108  	for _, f := range files {
   109  		if err := os.Remove(f); err != nil {
   110  			// this could be a race condition.
   111  			// conversely, on Windows, err may be nil and
   112  			// the file not deleted if anyone has it open.
   113  			u.logger.Printf("%v failed to remove %s", err, f)
   114  		}
   115  	}
   116  }
   117  
   118  // createReport creates local and upload report files by
   119  // combining all the count files for the expiryDate, and
   120  // returns the upload report file's path.
   121  // It may delete the count files once local and upload report
   122  // files are successfully created.
   123  func (u *uploader) createReport(start time.Time, expiryDate string, countFiles []string, lastWeek string) (string, error) {
   124  	uploadOK := true
   125  	mode, asof := u.dir.Mode()
   126  	if mode != "on" {
   127  		u.logger.Printf("No upload config or mode %q is not 'on'", mode)
   128  		uploadOK = false // no config, nothing to upload
   129  	}
   130  	if u.tooOld(expiryDate, u.startTime) {
   131  		u.logger.Printf("Expiry date %s is too old", expiryDate)
   132  		uploadOK = false
   133  	}
   134  	// If the mode is recorded with an asof date, don't upload if the report
   135  	// includes any data on or before the asof date.
   136  	if !asof.IsZero() && !asof.Before(start) {
   137  		u.logger.Printf("As-of date %s is not before start %s", asof, start)
   138  		uploadOK = false
   139  	}
   140  	// TODO(rfindley): check that all the x.Meta are consistent for GOOS, GOARCH, etc.
   141  	report := &telemetry.Report{
   142  		Config:   u.configVersion,
   143  		X:        computeRandom(), // json encodes all the bits
   144  		Week:     expiryDate,
   145  		LastWeek: lastWeek,
   146  	}
   147  	if report.X > u.config.SampleRate && u.config.SampleRate > 0 {
   148  		u.logger.Printf("X: %f > SampleRate:%f, not uploadable", report.X, u.config.SampleRate)
   149  		uploadOK = false
   150  	}
   151  	var succeeded bool
   152  	for _, f := range countFiles {
   153  		fok := false
   154  		x, err := u.parseCountFile(f)
   155  		if err != nil {
   156  			u.logger.Printf("Unparseable count file %s: %v", filepath.Base(f), err)
   157  			continue
   158  		}
   159  		prog := findProgReport(x.Meta, report)
   160  		for k, v := range x.Count {
   161  			if counter.IsStackCounter(k) {
   162  				// stack
   163  				prog.Stacks[k] += int64(v)
   164  			} else {
   165  				// counter
   166  				prog.Counters[k] += int64(v)
   167  			}
   168  			succeeded = true
   169  			fok = true
   170  		}
   171  		if !fok {
   172  			u.logger.Printf("no counters found in %s", f)
   173  		}
   174  	}
   175  	if !succeeded {
   176  		return "", fmt.Errorf("none of the %d count files for %s contained counters", len(countFiles), expiryDate)
   177  	}
   178  	// 1. generate the local report
   179  	localContents, err := json.MarshalIndent(report, "", " ")
   180  	if err != nil {
   181  		return "", fmt.Errorf("failed to marshal report for %s: %v", expiryDate, err)
   182  	}
   183  	// check that the report can be read back
   184  	// TODO(pjw): remove for production?
   185  	var report2 telemetry.Report
   186  	if err := json.Unmarshal(localContents, &report2); err != nil {
   187  		return "", fmt.Errorf("failed to unmarshal local report for %s: %v", expiryDate, err)
   188  	}
   189  
   190  	var uploadContents []byte
   191  	if uploadOK {
   192  		// 2. create the uploadable version
   193  		cfg := config.NewConfig(u.config)
   194  		upload := &telemetry.Report{
   195  			Week:     report.Week,
   196  			LastWeek: report.LastWeek,
   197  			X:        report.X,
   198  			Config:   report.Config,
   199  		}
   200  		for _, p := range report.Programs {
   201  			// does the uploadConfig want this program?
   202  			// if so, copy over the Stacks and Counters
   203  			// that the uploadConfig mentions.
   204  			if !cfg.HasGoVersion(p.GoVersion) || !cfg.HasProgram(p.Program) || !cfg.HasVersion(p.Program, p.Version) {
   205  				continue
   206  			}
   207  			x := &telemetry.ProgramReport{
   208  				Program:   p.Program,
   209  				Version:   p.Version,
   210  				GOOS:      p.GOOS,
   211  				GOARCH:    p.GOARCH,
   212  				GoVersion: p.GoVersion,
   213  				Counters:  make(map[string]int64),
   214  				Stacks:    make(map[string]int64),
   215  			}
   216  			upload.Programs = append(upload.Programs, x)
   217  			for k, v := range p.Counters {
   218  				if cfg.HasCounter(p.Program, k) && report.X <= cfg.Rate(p.Program, k) {
   219  					x.Counters[k] = v
   220  				}
   221  			}
   222  			// and the same for Stacks
   223  			// this can be made more efficient, when it matters
   224  			for k, v := range p.Stacks {
   225  				before, _, _ := strings.Cut(k, "\n")
   226  				if cfg.HasStack(p.Program, before) && report.X <= cfg.Rate(p.Program, before) {
   227  					x.Stacks[k] = v
   228  				}
   229  			}
   230  		}
   231  
   232  		uploadContents, err = json.MarshalIndent(upload, "", " ")
   233  		if err != nil {
   234  			return "", fmt.Errorf("failed to marshal upload report for %s: %v", expiryDate, err)
   235  		}
   236  	}
   237  	localFileName := filepath.Join(u.dir.LocalDir(), "local."+expiryDate+".json")
   238  	uploadFileName := filepath.Join(u.dir.LocalDir(), expiryDate+".json")
   239  
   240  	/* Prepare to write files */
   241  	// if either file exists, someone has been here ahead of us
   242  	// (there is still a race, but this check shortens the open window)
   243  	if _, err := os.Stat(localFileName); err == nil {
   244  		u.deleteFiles(countFiles)
   245  		return "", fmt.Errorf("local report %s already exists", localFileName)
   246  	}
   247  	if _, err := os.Stat(uploadFileName); err == nil {
   248  		u.deleteFiles(countFiles)
   249  		return "", fmt.Errorf("report %s already exists", uploadFileName)
   250  	}
   251  	// write the uploadable file
   252  	var errUpload, errLocal error
   253  	if uploadOK {
   254  		_, errUpload = exclusiveWrite(uploadFileName, uploadContents)
   255  	}
   256  	// write the local file
   257  	_, errLocal = exclusiveWrite(localFileName, localContents)
   258  	/*  Wrote the files */
   259  
   260  	// even though these errors won't occur, what should happen
   261  	// if errUpload == nil and it is ok to upload, and errLocal != nil?
   262  	if errLocal != nil {
   263  		return "", fmt.Errorf("failed to write local file %s (%v)", localFileName, errLocal)
   264  	}
   265  	if errUpload != nil {
   266  		return "", fmt.Errorf("failed to write upload file %s (%v)", uploadFileName, errUpload)
   267  	}
   268  	u.logger.Printf("Created %s, deleting %d count files", filepath.Base(uploadFileName), len(countFiles))
   269  	u.deleteFiles(countFiles)
   270  	if uploadOK {
   271  		return uploadFileName, nil
   272  	}
   273  	return "", nil
   274  }
   275  
   276  // exclusiveWrite attempts to create filename exclusively, and if successful,
   277  // writes content to the resulting file handle.
   278  //
   279  // It returns a boolean indicating whether the exclusive handle was acquired,
   280  // and an error indicating whether the operation succeeded.
   281  // If the file already exists, exclusiveWrite returns (false, nil).
   282  func exclusiveWrite(filename string, content []byte) (_ bool, rerr error) {
   283  	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
   284  	if err != nil {
   285  		if os.IsExist(err) {
   286  			return false, nil
   287  		}
   288  		return false, err
   289  	}
   290  	defer func() {
   291  		if err := f.Close(); err != nil && rerr == nil {
   292  			rerr = err
   293  		}
   294  	}()
   295  	if _, err := f.Write(content); err != nil {
   296  		return false, err
   297  	}
   298  	return true, nil
   299  }
   300  
   301  // return an existing ProgremReport, or create anew
   302  func findProgReport(meta map[string]string, report *telemetry.Report) *telemetry.ProgramReport {
   303  	for _, prog := range report.Programs {
   304  		if prog.Program == meta["Program"] && prog.Version == meta["Version"] &&
   305  			prog.GoVersion == meta["GoVersion"] && prog.GOOS == meta["GOOS"] &&
   306  			prog.GOARCH == meta["GOARCH"] {
   307  			return prog
   308  		}
   309  	}
   310  	prog := telemetry.ProgramReport{
   311  		Program:   meta["Program"],
   312  		Version:   meta["Version"],
   313  		GoVersion: meta["GoVersion"],
   314  		GOOS:      meta["GOOS"],
   315  		GOARCH:    meta["GOARCH"],
   316  		Counters:  make(map[string]int64),
   317  		Stacks:    make(map[string]int64),
   318  	}
   319  	report.Programs = append(report.Programs, &prog)
   320  	return &prog
   321  }
   322  
   323  // computeRandom returns a cryptographic random float64 in the range [0, 1],
   324  // with 52 bits of precision.
   325  func computeRandom() float64 {
   326  	for {
   327  		b := make([]byte, 8)
   328  		_, err := rand.Read(b)
   329  		if err != nil {
   330  			panic(fmt.Sprintf("rand.Read failed: %v", err))
   331  		}
   332  		// and turn it into a float64
   333  		x := math.Float64frombits(binary.LittleEndian.Uint64(b))
   334  		if math.IsNaN(x) || math.IsInf(x, 0) {
   335  			continue
   336  		}
   337  		x = math.Abs(x)
   338  		if x < 0x1p-1000 { // avoid underflow patterns
   339  			continue
   340  		}
   341  		frac, _ := math.Frexp(x) // 52 bits of randomness
   342  		return frac*2 - 1
   343  	}
   344  }
   345
View as plain text