findwork.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package upload
     6  
     7  import (
     8  	"os"
     9  	"path/filepath"
    10  	"strings"
    11  )
    12  
    13  // files to handle
    14  type work struct {
    15  	// absolute file names
    16  	countfiles []string // count files to process
    17  	readyfiles []string // old reports to upload
    18  	// relative names
    19  	uploaded map[string]bool // reports that have been uploaded
    20  }
    21  
    22  // find all the files that look like counter files or reports
    23  // that need to be uploaded. (There may be unexpected leftover files
    24  // and uploading is supposed to be idempotent.)
    25  func (u *uploader) findWork() work {
    26  	localdir, uploaddir := u.dir.LocalDir(), u.dir.UploadDir()
    27  	var ans work
    28  	fis, err := os.ReadDir(localdir)
    29  	if err != nil {
    30  		u.logger.Printf("Could not find work: failed to read local dir %s: %v", localdir, err)
    31  		return ans
    32  	}
    33  
    34  	mode, asof := u.dir.Mode()
    35  	u.logger.Printf("Finding work: mode %s asof %s", mode, asof)
    36  
    37  	// count files end in .v1.count
    38  	// reports end in .json. If they are not to be uploaded they
    39  	// start with local.
    40  	for _, fi := range fis {
    41  		if strings.HasSuffix(fi.Name(), ".v1.count") {
    42  			fname := filepath.Join(localdir, fi.Name())
    43  			_, expiry, err := u.counterDateSpan(fname)
    44  			switch {
    45  			case err != nil:
    46  				u.logger.Printf("Error reading expiry for count file %s: %v", fi.Name(), err)
    47  			case expiry.After(u.startTime):
    48  				u.logger.Printf("Skipping count file %s: still active", fi.Name())
    49  			default:
    50  				u.logger.Printf("Collecting count file %s", fi.Name())
    51  				ans.countfiles = append(ans.countfiles, fname)
    52  			}
    53  		} else if strings.HasPrefix(fi.Name(), "local.") {
    54  			// skip
    55  		} else if strings.HasSuffix(fi.Name(), ".json") && mode == "on" {
    56  			// Collect reports that are ready for upload.
    57  			reportDate := u.uploadReportDate(fi.Name())
    58  			if !asof.IsZero() && !reportDate.IsZero() {
    59  				// If both the mode asof date and the report date are present, do the
    60  				// right thing...
    61  				//
    62  				// (see https://github.com/golang/go/issues/63142#issuecomment-1734025130)
    63  				if asof.Before(reportDate) {
    64  					// Note: since this report was created after telemetry was enabled,
    65  					// we can only assume that the process that created it checked that
    66  					// the counter data contained therein was all from after the asof
    67  					// date.
    68  					//
    69  					// TODO(rfindley): store the begin date in reports, so that we can
    70  					// verify this assumption.
    71  					u.logger.Printf("Uploadable: %s", fi.Name())
    72  					ans.readyfiles = append(ans.readyfiles, filepath.Join(localdir, fi.Name()))
    73  				}
    74  			} else {
    75  				// ...otherwise fall back on the old behavior of uploading all
    76  				// unuploaded files.
    77  				//
    78  				// TODO(rfindley): invert this logic following more testing. We
    79  				// should only upload if we know both the asof date and the report
    80  				// date, and they are acceptable.
    81  				u.logger.Printf("Uploadable (missing date): %s", fi.Name())
    82  				ans.readyfiles = append(ans.readyfiles, filepath.Join(localdir, fi.Name()))
    83  			}
    84  		}
    85  	}
    86  
    87  	fis, err = os.ReadDir(uploaddir)
    88  	if err != nil {
    89  		os.MkdirAll(uploaddir, 0777)
    90  		return ans
    91  	}
    92  	// There should be only one of these per day; maybe sometime
    93  	// we'll want to clean the directory.
    94  	ans.uploaded = make(map[string]bool)
    95  	for _, fi := range fis {
    96  		if strings.HasSuffix(fi.Name(), ".json") {
    97  			u.logger.Printf("Already uploaded: %s", fi.Name())
    98  			ans.uploaded[fi.Name()] = true
    99  		}
   100  	}
   101  	return ans
   102  }
   103
View as plain text