pack.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Distpack creates the tgz and zip files for a Go distribution.
     6  // It writes into GOROOT/pkg/distpack:
     7  //
     8  //   - a binary distribution (tgz or zip) for the current GOOS and GOARCH
     9  //   - a source distribution that is independent of GOOS/GOARCH
    10  //   - the module mod, info, and zip files for a distribution in module form
    11  //     (as used by GOTOOLCHAIN support in the go command).
    12  //
    13  // Distpack is typically invoked by the -distpack flag to make.bash.
    14  // A cross-compiled distribution for goos/goarch can be built using:
    15  //
    16  //	GOOS=goos GOARCH=goarch ./make.bash -distpack
    17  //
    18  // To test that the module downloads are usable with the go command:
    19  //
    20  //	./make.bash -distpack
    21  //	mkdir -p /tmp/goproxy/golang.org/toolchain/
    22  //	ln -sf $(pwd)/../pkg/distpack /tmp/goproxy/golang.org/toolchain/@v
    23  //	GOPROXY=file:///tmp/goproxy GOTOOLCHAIN=$(sed 1q ../VERSION) gotip version
    24  //
    25  // gotip can be replaced with an older released Go version once there is one.
    26  // It just can't be the one make.bash built, because it knows it is already that
    27  // version and will skip the download.
    28  package main
    29  
    30  import (
    31  	"archive/tar"
    32  	"archive/zip"
    33  	"compress/flate"
    34  	"compress/gzip"
    35  	"crypto/sha256"
    36  	"flag"
    37  	"fmt"
    38  	"io"
    39  	"io/fs"
    40  	"log"
    41  	"os"
    42  	"path"
    43  	"path/filepath"
    44  	"runtime"
    45  	"strings"
    46  	"time"
    47  
    48  	"cmd/internal/telemetry/counter"
    49  )
    50  
    51  func usage() {
    52  	fmt.Fprintf(os.Stderr, "usage: distpack\n")
    53  	os.Exit(2)
    54  }
    55  
    56  const (
    57  	modPath          = "golang.org/toolchain"
    58  	modVersionPrefix = "v0.0.1"
    59  )
    60  
    61  var (
    62  	goroot     string
    63  	gohostos   string
    64  	gohostarch string
    65  	goos       string
    66  	goarch     string
    67  )
    68  
    69  func main() {
    70  	log.SetPrefix("distpack: ")
    71  	log.SetFlags(0)
    72  	counter.Open()
    73  	flag.Usage = usage
    74  	flag.Parse()
    75  	counter.Inc("distpack/invocations")
    76  	counter.CountFlags("distpack/flag:", *flag.CommandLine)
    77  	if flag.NArg() != 0 {
    78  		usage()
    79  	}
    80  
    81  	// Load context.
    82  	goroot = runtime.GOROOT()
    83  	if goroot == "" {
    84  		log.Fatalf("missing $GOROOT")
    85  	}
    86  	gohostos = runtime.GOOS
    87  	gohostarch = runtime.GOARCH
    88  	goos = os.Getenv("GOOS")
    89  	if goos == "" {
    90  		goos = gohostos
    91  	}
    92  	goarch = os.Getenv("GOARCH")
    93  	if goarch == "" {
    94  		goarch = gohostarch
    95  	}
    96  	goosUnderGoarch := goos + "_" + goarch
    97  	goosDashGoarch := goos + "-" + goarch
    98  	exe := ""
    99  	if goos == "windows" {
   100  		exe = ".exe"
   101  	}
   102  	version, versionTime := readVERSION(goroot)
   103  
   104  	// Start with files from GOROOT, filtering out non-distribution files.
   105  	base, err := NewArchive(goroot)
   106  	if err != nil {
   107  		log.Fatal(err)
   108  	}
   109  	base.SetTime(versionTime)
   110  	base.SetMode(mode)
   111  	base.Remove(
   112  		".git/**",
   113  		".gitattributes",
   114  		".github/**",
   115  		".gitignore",
   116  		".jj/**",
   117  		"VERSION.cache",
   118  		"misc/cgo/*/_obj/**",
   119  		"**/.DS_Store",
   120  		"**/*.exe~", // go.dev/issue/23894
   121  		// Generated during make.bat/make.bash.
   122  		"src/cmd/dist/dist",
   123  		"src/cmd/dist/dist.exe",
   124  	)
   125  
   126  	// The source distribution removes files generated during the release build.
   127  	// See ../dist/build.go's deptab.
   128  	srcArch := base.Clone()
   129  	srcArch.Remove(
   130  		"bin/**",
   131  		"pkg/**",
   132  
   133  		// Generated during cmd/dist. See ../dist/build.go:/gentab.
   134  		"src/cmd/go/internal/cfg/zdefaultcc.go",
   135  		"src/go/build/zcgo.go",
   136  		"src/internal/runtime/sys/zversion.go",
   137  		"src/time/tzdata/zzipdata.go",
   138  
   139  		// Generated during cmd/dist by bootstrapBuildTools.
   140  		"src/cmd/cgo/zdefaultcc.go",
   141  		"src/cmd/internal/objabi/zbootstrap.go",
   142  		"src/internal/buildcfg/zbootstrap.go",
   143  
   144  		// Generated by earlier versions of cmd/dist .
   145  		"src/cmd/go/internal/cfg/zosarch.go",
   146  	)
   147  	srcArch.AddPrefix("go")
   148  	testSrc(srcArch)
   149  
   150  	// The binary distribution includes only a subset of bin and pkg.
   151  	binArch := base.Clone()
   152  	binArch.Filter(func(name string) bool {
   153  		// Discard bin/ for now, will add back later.
   154  		if strings.HasPrefix(name, "bin/") {
   155  			return false
   156  		}
   157  		// Discard most of pkg.
   158  		if strings.HasPrefix(name, "pkg/") {
   159  			// Keep pkg/include.
   160  			if strings.HasPrefix(name, "pkg/include/") {
   161  				return true
   162  			}
   163  			// Discard other pkg except pkg/tool.
   164  			if !strings.HasPrefix(name, "pkg/tool/") {
   165  				return false
   166  			}
   167  			// Inside pkg/tool, keep only $GOOS_$GOARCH.
   168  			if !strings.HasPrefix(name, "pkg/tool/"+goosUnderGoarch+"/") {
   169  				return false
   170  			}
   171  			// Inside pkg/tool/$GOOS_$GOARCH, keep only tools needed for build actions.
   172  			switch strings.TrimSuffix(path.Base(name), ".exe") {
   173  			default:
   174  				return false
   175  			// Keep in sync with toolsIncludedInDistpack in cmd/dist/build.go.
   176  			case "asm", "cgo", "compile", "cover", "export", "fix", "link", "preprofile", "vet":
   177  			}
   178  		}
   179  		return true
   180  	})
   181  
   182  	// Add go and gofmt to bin, using cross-compiled binaries
   183  	// if this is a cross-compiled distribution.
   184  	// Keep in sync with binExesIncludedInDistpack in cmd/dist/build.go.
   185  	binExes := []string{
   186  		"go",
   187  		"gofmt",
   188  	}
   189  	crossBin := "bin"
   190  	if goos != gohostos || goarch != gohostarch {
   191  		crossBin = "bin/" + goosUnderGoarch
   192  	}
   193  	for _, b := range binExes {
   194  		name := "bin/" + b + exe
   195  		src := filepath.Join(goroot, crossBin, b+exe)
   196  		info, err := os.Stat(src)
   197  		if err != nil {
   198  			log.Fatal(err)
   199  		}
   200  		binArch.Add(name, src, info)
   201  	}
   202  	binArch.Sort()
   203  	binArch.SetTime(versionTime) // fix added files
   204  	binArch.SetMode(mode)        // fix added files
   205  
   206  	zipArch := binArch.Clone()
   207  	zipArch.AddPrefix("go")
   208  	testZip(zipArch)
   209  
   210  	// The module distribution is the binary distribution with unnecessary files removed
   211  	// and file names using the necessary prefix for the module.
   212  	modArch := binArch.Clone()
   213  	modArch.Remove(
   214  		"api/**",
   215  		"doc/**",
   216  		"misc/**",
   217  		"test/**",
   218  	)
   219  	modVers := modVersionPrefix + "-" + version + "." + goosDashGoarch
   220  	modArch.AddPrefix(modPath + "@" + modVers)
   221  	modArch.RenameGoMod()
   222  	modArch.Sort()
   223  	testMod(modArch)
   224  
   225  	// distpack returns the full path to name in the distpack directory.
   226  	distpack := func(name string) string {
   227  		return filepath.Join(goroot, "pkg/distpack", name)
   228  	}
   229  	if err := os.MkdirAll(filepath.Join(goroot, "pkg/distpack"), 0777); err != nil {
   230  		log.Fatal(err)
   231  	}
   232  
   233  	writeTgz(distpack(version+".src.tar.gz"), srcArch)
   234  
   235  	if goos == "windows" {
   236  		writeZip(distpack(version+"."+goos+"-"+goarch+".zip"), zipArch)
   237  	} else {
   238  		writeTgz(distpack(version+"."+goos+"-"+goarch+".tar.gz"), zipArch)
   239  	}
   240  
   241  	writeZip(distpack(modVers+".zip"), modArch)
   242  	writeFile(distpack(modVers+".mod"),
   243  		[]byte(fmt.Sprintf("module %s\n", modPath)))
   244  	writeFile(distpack(modVers+".info"),
   245  		[]byte(fmt.Sprintf("{%q:%q, %q:%q}\n",
   246  			"Version", modVers,
   247  			"Time", versionTime.Format(time.RFC3339))))
   248  }
   249  
   250  // mode computes the mode for the given file name.
   251  func mode(name string, _ fs.FileMode) fs.FileMode {
   252  	if strings.HasPrefix(name, "bin/") ||
   253  		strings.HasPrefix(name, "pkg/tool/") ||
   254  		strings.HasSuffix(name, ".bash") ||
   255  		strings.HasSuffix(name, ".sh") ||
   256  		strings.HasSuffix(name, ".pl") ||
   257  		strings.HasSuffix(name, ".rc") {
   258  		return 0o755
   259  	} else if ok, _ := amatch("**/go_?*_?*_exec", name); ok {
   260  		return 0o755
   261  	}
   262  	return 0o644
   263  }
   264  
   265  // readVERSION reads the VERSION file.
   266  // The first line of the file is the Go version.
   267  // Additional lines are 'key value' pairs setting other data.
   268  // The only valid key at the moment is 'time', which sets the modification time for file archives.
   269  func readVERSION(goroot string) (version string, t time.Time) {
   270  	data, err := os.ReadFile(filepath.Join(goroot, "VERSION"))
   271  	if err != nil {
   272  		log.Fatal(err)
   273  	}
   274  	version, rest, _ := strings.Cut(string(data), "\n")
   275  	for line := range strings.SplitSeq(rest, "\n") {
   276  		f := strings.Fields(line)
   277  		if len(f) == 0 {
   278  			continue
   279  		}
   280  		switch f[0] {
   281  		default:
   282  			log.Fatalf("VERSION: unexpected line: %s", line)
   283  		case "time":
   284  			if len(f) != 2 {
   285  				log.Fatalf("VERSION: unexpected time line: %s", line)
   286  			}
   287  			t, err = time.ParseInLocation(time.RFC3339, f[1], time.UTC)
   288  			if err != nil {
   289  				log.Fatalf("VERSION: bad time: %s", err)
   290  			}
   291  		}
   292  	}
   293  	return version, t
   294  }
   295  
   296  // writeFile writes a file with the given name and data or fatals.
   297  func writeFile(name string, data []byte) {
   298  	if err := os.WriteFile(name, data, 0666); err != nil {
   299  		log.Fatal(err)
   300  	}
   301  	reportHash(name)
   302  }
   303  
   304  // check panics if err is not nil. Otherwise it returns x.
   305  // It is only meant to be used in a function that has deferred
   306  // a function to recover appropriately from the panic.
   307  func check[T any](x T, err error) T {
   308  	check1(err)
   309  	return x
   310  }
   311  
   312  // check1 panics if err is not nil.
   313  // It is only meant to be used in a function that has deferred
   314  // a function to recover appropriately from the panic.
   315  func check1(err error) {
   316  	if err != nil {
   317  		panic(err)
   318  	}
   319  }
   320  
   321  // writeTgz writes the archive in tgz form to the file named name.
   322  func writeTgz(name string, a *Archive) {
   323  	out, err := os.Create(name)
   324  	if err != nil {
   325  		log.Fatal(err)
   326  	}
   327  
   328  	var f File
   329  	defer func() {
   330  		if err := recover(); err != nil {
   331  			extra := ""
   332  			if f.Name != "" {
   333  				extra = " " + f.Name
   334  			}
   335  			log.Fatalf("writing %s%s: %v", name, extra, err)
   336  		}
   337  	}()
   338  
   339  	zw := check(gzip.NewWriterLevel(out, gzip.BestCompression))
   340  	tw := tar.NewWriter(zw)
   341  
   342  	// Find the mode and mtime to use for directory entries,
   343  	// based on the mode and mtime of the first file we see.
   344  	// We know that modes and mtimes are uniform across the archive.
   345  	var dirMode fs.FileMode
   346  	var mtime time.Time
   347  	for _, f := range a.Files {
   348  		dirMode = fs.ModeDir | f.Mode | (f.Mode&0444)>>2 // copy r bits down to x bits
   349  		mtime = f.Time
   350  		break
   351  	}
   352  
   353  	// mkdirAll ensures that the tar file contains directory
   354  	// entries for dir and all its parents. Some programs reading
   355  	// these tar files expect that. See go.dev/issue/61862.
   356  	haveDir := map[string]bool{".": true}
   357  	var mkdirAll func(string)
   358  	mkdirAll = func(dir string) {
   359  		if dir == "/" {
   360  			panic("mkdirAll /")
   361  		}
   362  		if haveDir[dir] {
   363  			return
   364  		}
   365  		haveDir[dir] = true
   366  		mkdirAll(path.Dir(dir))
   367  		df := &File{
   368  			Name: dir + "/",
   369  			Time: mtime,
   370  			Mode: dirMode,
   371  		}
   372  		h := check(tar.FileInfoHeader(df.Info(), ""))
   373  		h.Name = dir + "/"
   374  		if err := tw.WriteHeader(h); err != nil {
   375  			panic(err)
   376  		}
   377  	}
   378  
   379  	for _, f = range a.Files {
   380  		h := check(tar.FileInfoHeader(f.Info(), ""))
   381  		mkdirAll(path.Dir(f.Name))
   382  		h.Name = f.Name
   383  		if err := tw.WriteHeader(h); err != nil {
   384  			panic(err)
   385  		}
   386  		r := check(os.Open(f.Src))
   387  		check(io.Copy(tw, r))
   388  		check1(r.Close())
   389  	}
   390  	f.Name = ""
   391  	check1(tw.Close())
   392  	check1(zw.Close())
   393  	check1(out.Close())
   394  	reportHash(name)
   395  }
   396  
   397  // writeZip writes the archive in zip form to the file named name.
   398  func writeZip(name string, a *Archive) {
   399  	out, err := os.Create(name)
   400  	if err != nil {
   401  		log.Fatal(err)
   402  	}
   403  
   404  	var f File
   405  	defer func() {
   406  		if err := recover(); err != nil {
   407  			extra := ""
   408  			if f.Name != "" {
   409  				extra = " " + f.Name
   410  			}
   411  			log.Fatalf("writing %s%s: %v", name, extra, err)
   412  		}
   413  	}()
   414  
   415  	zw := zip.NewWriter(out)
   416  	zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
   417  		return flate.NewWriter(out, flate.BestCompression)
   418  	})
   419  	for _, f = range a.Files {
   420  		h := check(zip.FileInfoHeader(f.Info()))
   421  		h.Name = f.Name
   422  		h.Method = zip.Deflate
   423  		w := check(zw.CreateHeader(h))
   424  		r := check(os.Open(f.Src))
   425  		check(io.Copy(w, r))
   426  		check1(r.Close())
   427  	}
   428  	f.Name = ""
   429  	check1(zw.Close())
   430  	check1(out.Close())
   431  	reportHash(name)
   432  }
   433  
   434  func reportHash(name string) {
   435  	f, err := os.Open(name)
   436  	if err != nil {
   437  		log.Fatal(err)
   438  	}
   439  	h := sha256.New()
   440  	io.Copy(h, f)
   441  	f.Close()
   442  	fmt.Printf("distpack: %x %s\n", h.Sum(nil)[:8], filepath.Base(name))
   443  }
   444
View as plain text