input.go

     1  // Copyright 2026 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"cmp"
     9  	"errors"
    10  	"go/ast"
    11  	"go/token"
    12  	"go/types"
    13  	"io/fs"
    14  	"log"
    15  	"os"
    16  	"path/filepath"
    17  	"slices"
    18  	"strings"
    19  
    20  	"golang.org/x/tools/go/packages"
    21  	"gopkg.in/yaml.v3"
    22  
    23  	"simd/archsimd/_gen/gentools"
    24  )
    25  
    26  // buildConfigs are the configurations we report API coverage for. The API is spread
    27  // across files selected by build constraints, and the only authority on which
    28  // files belong to which configuration is the go command -- so rather than
    29  // interpreting //go:build lines here, we ask go/packages once per configuration
    30  // and use the file set it hands back.
    31  //
    32  // SVE is a target in SPEC-TRANSITION.md's sense but not a GOARCH: its files are
    33  // built for arm64 like NEON's. It separates out along the generator dimension
    34  // instead, because simdgen records its -arch in the generated-by header.
    35  var buildConfigs = []struct{ name, goos, goarch string }{
    36  	{"amd64", "linux", "amd64"},
    37  	{"arm64", "linux", "arm64"},
    38  	{"wasm", "js", "wasm"},
    39  	// Package simd falls back to a pure-Go implementation on architectures with
    40  	// no SIMD support. 386 is an arbitrary representative of those.
    41  	{"portable", "linux", "386"},
    42  }
    43  
    44  // A decl is one exported function or method declaration in the public API.
    45  type decl struct {
    46  	pkg     string   // "simd" or "archsimd"
    47  	file    string   // path relative to the simd directory
    48  	gen     string   // generator that emitted it, or "hand-written"
    49  	targets []string // names of build configurations it compiles in
    50  	recv    string   // receiver type name, empty for plain functions
    51  	name    string
    52  	in      []param
    53  	out     []param
    54  
    55  	doc    string // doc comment text, comment markers stripped
    56  	hasSep bool   // doc contains a bare "//" separator line
    57  }
    58  
    59  type param struct{ name, typ string }
    60  
    61  // The grains the report counts at, coarsest last. Spec describes an operation
    62  // once, the API declares it once per target, and a name collapses receivers
    63  // too; the gaps between the three are diagnostic (SPEC-TRANSITION.md §1.2).
    64  type (
    65  	// methodKey is the (receiver, name) key spec is unique on.
    66  	methodKey struct{ recv, name string }
    67  
    68  	// archMethodKey is a method as it appears in one build configuration.
    69  	//
    70  	// It carries the package because simd and archsimd share their scalable
    71  	// type names.
    72  	archMethodKey struct {
    73  		target string
    74  		pkg    string
    75  		methodKey
    76  	}
    77  
    78  	// declKey identifies one declaration in the source, which is what collapses
    79  	// the several configurations it may build in.
    80  	declKey struct {
    81  		file string
    82  		methodKey
    83  	}
    84  )
    85  
    86  // A fact is one declaration as seen in one build configuration: the grain of
    87  // the fact table, and the row type every figure in the report pivots over.
    88  type fact struct {
    89  	*decl
    90  	target string
    91  }
    92  
    93  func (d *decl) methodKey() methodKey        { return methodKey{d.recv, d.name} }
    94  func (f fact) declKey() declKey             { return declKey{f.file, f.methodKey()} }
    95  func (f fact) archMethodKey() archMethodKey { return archMethodKey{f.target, f.pkg, f.methodKey()} }
    96  
    97  // tool strips the invocation from a generator label, so that simdgen's three
    98  // runs count as one tool.
    99  func (f fact) tool() string { t, _, _ := strings.Cut(f.gen, "/"); return t }
   100  
   101  // nonSpecOp reports whether name is an operation that will never come from spec:
   102  // they are Go conventions (String, Len) or architecture bridge operations
   103  // (ToArch, FromArch). They are excluded from the fact table, and
   104  // so from every denominator. See SPEC-TRANSITION.md §2.7.
   105  func nonSpecOp(name string) bool {
   106  	return name == "String" || name == "Len" || name == "ToArch" || strings.HasSuffix(name, "FromArch")
   107  }
   108  
   109  // loadOverlay builds a packages.Config.Overlay map from the scratch overlay
   110  // directory if one is active.
   111  func loadOverlay(genOpts *gentools.Options) map[string][]byte {
   112  	if genOpts == nil {
   113  		return nil
   114  	}
   115  	dir := genOpts.OverlayDir()
   116  	if dir == "" {
   117  		return nil
   118  	}
   119  	root := filepath.Join(dir, "src")
   120  	overlay := make(map[string][]byte)
   121  	_ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
   122  		if err != nil || d.IsDir() {
   123  			return nil
   124  		}
   125  		rel, err := filepath.Rel(root, path)
   126  		if err != nil {
   127  			return nil
   128  		}
   129  		slashRel := filepath.ToSlash(rel)
   130  		if !strings.HasPrefix(slashRel, "simd/") {
   131  			return nil
   132  		}
   133  		b, err := genOpts.ReadFile(rel)
   134  		if err != nil {
   135  			return nil
   136  		}
   137  		gorootPath := filepath.Join(genOpts.GOROOT, "src", filepath.FromSlash(rel))
   138  		overlay[gorootPath] = b
   139  		return nil
   140  	})
   141  	return overlay
   142  }
   143  
   144  func loadAPI(genOpts *gentools.Options, simdDir string) []*decl {
   145  	fset := token.NewFileSet()
   146  	byKey := map[declKey]*decl{}
   147  	overlay := loadOverlay(genOpts)
   148  
   149  	for _, bc := range buildConfigs {
   150  		cfg := &packages.Config{
   151  			// Syntax only: we compare declarations across configurations, which
   152  			// type-checking would not help with and would slow down. See
   153  			// SPEC-TRANSITION.md §2.4.
   154  			Mode:    packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedSyntax,
   155  			Dir:     simdDir,
   156  			Fset:    fset,
   157  			Env:     append(os.Environ(), "GOOS="+bc.goos, "GOARCH="+bc.goarch, "GOEXPERIMENT=simd"),
   158  			Overlay: overlay,
   159  		}
   160  		pkgs, err := packages.Load(cfg, ".", "./archsimd")
   161  		if err != nil {
   162  			log.Fatalf("loading simd packages for %s/%s: %s", bc.goos, bc.goarch, err)
   163  		}
   164  		for _, p := range pkgs {
   165  			for _, e := range p.Errors {
   166  				log.Fatalf("loading %s for %s/%s: %s", p.PkgPath, bc.goos, bc.goarch, e)
   167  			}
   168  			for _, f := range p.Syntax {
   169  				file := fset.Position(f.Pos()).Filename
   170  				if rel, err := filepath.Rel(simdDir, file); err == nil {
   171  					file = filepath.ToSlash(rel)
   172  				}
   173  				gen := generatorOf(f)
   174  				for _, d := range f.Decls {
   175  					fd, ok := d.(*ast.FuncDecl)
   176  					if !ok || !fd.Name.IsExported() {
   177  						continue
   178  					}
   179  					dec := declOf(fd, p.Name, file, gen)
   180  					key := declKey{file, dec.methodKey()}
   181  					if prev, ok := byKey[key]; ok {
   182  						dec = prev
   183  					} else {
   184  						byKey[key] = dec
   185  					}
   186  					dec.targets = append(dec.targets, bc.name)
   187  				}
   188  			}
   189  		}
   190  	}
   191  
   192  	decls := make([]*decl, 0, len(byKey))
   193  	for _, d := range byKey {
   194  		slices.Sort(d.targets)
   195  		d.targets = slices.Compact(d.targets)
   196  		decls = append(decls, d)
   197  	}
   198  	slices.SortFunc(decls, func(a, b *decl) int {
   199  		return cmp.Or(cmp.Compare(a.file, b.file), cmp.Compare(a.recv, b.recv), cmp.Compare(a.name, b.name))
   200  	})
   201  	return decls
   202  }
   203  
   204  func declOf(fd *ast.FuncDecl, pkg, file, gen string) *decl {
   205  	d := &decl{pkg: pkg, file: file, gen: gen, name: fd.Name.Name}
   206  	if fd.Recv != nil && len(fd.Recv.List) == 1 {
   207  		// Receivers in this API are always by value and never generic.
   208  		if id, ok := fd.Recv.List[0].Type.(*ast.Ident); ok {
   209  			d.recv = id.Name
   210  		} else {
   211  			d.recv = types.ExprString(fd.Recv.List[0].Type)
   212  		}
   213  	}
   214  	d.in = fields(fd.Type.Params)
   215  	d.out = fields(fd.Type.Results)
   216  	if fd.Doc != nil {
   217  		var lines []string
   218  		for _, c := range fd.Doc.List {
   219  			lines = append(lines, c.Text)
   220  			if strings.TrimSpace(c.Text) == "//" {
   221  				d.hasSep = true
   222  			}
   223  		}
   224  		d.doc = strings.Join(lines, "\n")
   225  	}
   226  	return d
   227  }
   228  
   229  // fields flattens a parameter or result list, preserving declaration order and
   230  // expanding grouped names ("x, y Vec") into one param each.
   231  func fields(fl *ast.FieldList) []param {
   232  	if fl == nil {
   233  		return nil
   234  	}
   235  	var ps []param
   236  	for _, f := range fl.List {
   237  		typ := types.ExprString(f.Type)
   238  		if len(f.Names) == 0 {
   239  			ps = append(ps, param{"", typ})
   240  			continue
   241  		}
   242  		for _, n := range f.Names {
   243  			ps = append(ps, param{n.Name, typ})
   244  		}
   245  	}
   246  	return ps
   247  }
   248  
   249  // generatorOf returns the tool named in a "Code generated by" header, or
   250  // "hand-written" if there is none.
   251  //
   252  // Since simdgen runs separately per target, for simdgen this returns
   253  // "simdgen/<arch>", where arch is the -arch flag passed to simdgen.
   254  func generatorOf(f *ast.File) string {
   255  	for _, cg := range f.Comments {
   256  		for _, c := range cg.List {
   257  			if !strings.Contains(c.Text, "Code generated by") {
   258  				continue
   259  			}
   260  			for _, tool := range []string{"simdgen", "tmplgen", "wasmgen", "midway", "refgen"} {
   261  				if !strings.Contains(c.Text, tool) {
   262  					continue
   263  				}
   264  				if args := strings.Fields(c.Text); tool == "simdgen" {
   265  					for i, a := range args {
   266  						if a == "-arch" && i+1 < len(args) {
   267  							return tool + "/" + args[i+1]
   268  						}
   269  					}
   270  				}
   271  				return tool
   272  			}
   273  			return "generated"
   274  		}
   275  		break // only the leading comment block can carry the header
   276  	}
   277  	return "hand-written"
   278  }
   279  
   280  // readYAML parses one YAML document into a node tree and returns its root
   281  // content node. The generators' inputs carry custom tags (!sum, !string, and
   282  // the merge keys in comments.yaml), but a node tree keeps a tag as data rather
   283  // than trying to resolve it, so this reads them without knowing their schemas.
   284  //
   285  // A missing file counts as empty: these inputs are deleted by comments-yaml and cat-delete,
   286  // and their figures should read zero rather than crash once they are gone. A
   287  // malformed file is fatal, because that is a real problem with the input.
   288  func readYAML(path string) *yaml.Node {
   289  	b, err := os.ReadFile(path)
   290  	if errors.Is(err, fs.ErrNotExist) {
   291  		return nil
   292  	} else if err != nil {
   293  		log.Fatal(err)
   294  	}
   295  	var doc yaml.Node
   296  	if err := yaml.Unmarshal(b, &doc); err != nil {
   297  		log.Fatalf("%s: %s", path, err)
   298  	}
   299  	if doc.Kind == yaml.DocumentNode && len(doc.Content) == 1 {
   300  		return doc.Content[0]
   301  	}
   302  	return &doc
   303  }
   304  
   305  // eachEntry calls fn for every key/value pair of every mapping under n, depth
   306  // first.
   307  //
   308  // It does not follow aliases. comments.yaml merges a shared block into fourteen
   309  // receivers with "<<: *common_methods", and those entries are counted where
   310  // they are defined; expanding the merge would count them fifteen times over.
   311  func eachEntry(n *yaml.Node, fn func(key, val *yaml.Node)) {
   312  	if n == nil || n.Kind == yaml.AliasNode {
   313  		return
   314  	}
   315  	if n.Kind == yaml.MappingNode {
   316  		for i := 0; i+1 < len(n.Content); i += 2 {
   317  			fn(n.Content[i], n.Content[i+1])
   318  		}
   319  	}
   320  	for _, c := range n.Content {
   321  		eachEntry(c, fn)
   322  	}
   323  }
   324  
   325  // mapValue returns the value of key in a mapping node, or nil.
   326  func mapValue(n *yaml.Node, key string) *yaml.Node {
   327  	if n == nil || n.Kind != yaml.MappingNode {
   328  		return nil
   329  	}
   330  	for i := 0; i+1 < len(n.Content); i += 2 {
   331  		if n.Content[i].Value == key {
   332  			return n.Content[i+1]
   333  		}
   334  	}
   335  	return nil
   336  }
   337  
   338  // isString reports whether n is a string scalar, which in these files means a
   339  // documentation string. Block scalars ("|-") and quoted scalars both qualify.
   340  func isString(n *yaml.Node) bool {
   341  	return n != nil && n.Kind == yaml.ScalarNode && n.ShortTag() == "!!str"
   342  }
   343  
   344  // categoriesInventory holds the inventory of categories.yaml.
   345  type categoriesInventory struct {
   346  	allFields      map[string]int // field → count across all entries
   347  	exportedFields map[string]int // field → count across exported entries only
   348  	entries        int
   349  	uniq           int
   350  	exported       int
   351  	compilerOnly   int
   352  }
   353  
   354  func makeCategoriesInventory(genDir string) categoriesInventory {
   355  	r := categoriesInventory{
   356  		allFields:      map[string]int{},
   357  		exportedFields: map[string]int{},
   358  	}
   359  	names := map[string]bool{}
   360  	paths, _ := filepath.Glob(filepath.Join(genDir, "simdgen", "ops", "*", "categories.yaml"))
   361  	slices.Sort(paths)
   362  	for _, p := range paths {
   363  		root := readYAML(p)
   364  		if root == nil || root.Kind != yaml.SequenceNode {
   365  			continue
   366  		}
   367  		for _, entry := range root.Content {
   368  			if entry.Kind != yaml.MappingNode {
   369  				continue
   370  			}
   371  			goName := mapValue(entry, "go")
   372  			if goName == nil {
   373  				continue
   374  			}
   375  			r.entries++
   376  			names[goName.Value] = true
   377  			isExported := goName.Value != "" && !(goName.Value[0] >= 'a' && goName.Value[0] <= 'z')
   378  			hasNoTypes := mapValue(entry, "noTypes") != nil
   379  			if !isExported || hasNoTypes {
   380  				r.compilerOnly++
   381  			}
   382  			eachEntry(entry, func(k, _ *yaml.Node) {
   383  				r.allFields[k.Value]++
   384  				if isExported && !hasNoTypes {
   385  					r.exportedFields[k.Value]++
   386  				}
   387  			})
   388  		}
   389  	}
   390  	for n := range names {
   391  		r.uniq++
   392  		if n != "" && !(n[0] >= 'a' && n[0] <= 'z') {
   393  			r.exported++
   394  		}
   395  	}
   396  	return r
   397  }
   398  
   399  // countMidwayComments counts the doc-bearing entries in midway/comments.yaml:
   400  // every key whose value is a string, wherever it appears.
   401  func countMidwayComments(genDir string) (n int) {
   402  	eachEntry(readYAML(commentsPath(genDir)), func(_, v *yaml.Node) {
   403  		if isString(v) {
   404  			n++
   405  		}
   406  	})
   407  	return n
   408  }
   409  
   410  // commonMethods returns the method names defined in simd_emulated.go,
   411  // less [nonSpecOp]. That set is spec-common's scope, and the portable
   412  // core a new architecture implements first.
   413  func commonMethods(api []*decl) []string {
   414  	seen := map[string]bool{}
   415  	var names []string
   416  	for _, d := range api {
   417  		if d.file == "simd_emulated.go" && d.recv != "" && !nonSpecOp(d.name) {
   418  			if !seen[d.name] {
   419  				seen[d.name] = true
   420  				names = append(names, d.name)
   421  			}
   422  		}
   423  	}
   424  	slices.Sort(names)
   425  	return names
   426  }
   427  
   428  func commentsPath(genDir string) string {
   429  	return filepath.Join(genDir, "midway", "comments.yaml")
   430  }
   431
View as plain text