relnote.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package relnote supports working with release notes.
     6  //
     7  // Its main feature is the ability to merge Markdown fragments into a single
     8  // document. (See [Merge].)
     9  //
    10  // This package has minimal imports, so that it can be vendored into the
    11  // main go repo.
    12  package relnote
    13  
    14  import (
    15  	"bufio"
    16  	"bytes"
    17  	"errors"
    18  	"fmt"
    19  	"io"
    20  	"io/fs"
    21  	"path"
    22  	"regexp"
    23  	"slices"
    24  	"strconv"
    25  	"strings"
    26  
    27  	md "rsc.io/markdown"
    28  )
    29  
    30  // NewParser returns a properly configured Markdown parser.
    31  func NewParser() *md.Parser {
    32  	var p md.Parser
    33  	p.HeadingIDs = true
    34  	return &p
    35  }
    36  
    37  // CheckFragment reports problems in a release-note fragment.
    38  func CheckFragment(data string) error {
    39  	doc := NewParser().Parse(data)
    40  	// Check that the content of the document contains either a TODO or at least one sentence.
    41  	txt := ""
    42  	if len(doc.Blocks) > 0 {
    43  		txt = text(doc)
    44  	}
    45  	if !strings.Contains(txt, "TODO") && !strings.ContainsAny(txt, ".?!") {
    46  		return errors.New("File must contain a complete sentence or a TODO.")
    47  	}
    48  	return nil
    49  }
    50  
    51  // text returns all the text in a block, without any formatting.
    52  func text(b md.Block) string {
    53  	switch b := b.(type) {
    54  	case *md.Document:
    55  		return blocksText(b.Blocks)
    56  	case *md.Heading:
    57  		return text(b.Text)
    58  	case *md.Text:
    59  		return inlineText(b.Inline)
    60  	case *md.CodeBlock:
    61  		return strings.Join(b.Text, "\n")
    62  	case *md.HTMLBlock:
    63  		return strings.Join(b.Text, "\n")
    64  	case *md.List:
    65  		return blocksText(b.Items)
    66  	case *md.Item:
    67  		return blocksText(b.Blocks)
    68  	case *md.Empty:
    69  		return ""
    70  	case *md.Paragraph:
    71  		return text(b.Text)
    72  	case *md.Quote:
    73  		return blocksText(b.Blocks)
    74  	case *md.ThematicBreak:
    75  		return "---"
    76  	default:
    77  		panic(fmt.Sprintf("unknown block type %T", b))
    78  	}
    79  }
    80  
    81  // blocksText returns all the text in a slice of block nodes.
    82  func blocksText(bs []md.Block) string {
    83  	var d strings.Builder
    84  	for _, b := range bs {
    85  		io.WriteString(&d, text(b))
    86  		fmt.Fprintln(&d)
    87  	}
    88  	return d.String()
    89  }
    90  
    91  // inlineText returns all the next in a slice of inline nodes.
    92  func inlineText(ins []md.Inline) string {
    93  	var buf bytes.Buffer
    94  	for _, in := range ins {
    95  		in.PrintText(&buf)
    96  	}
    97  	return buf.String()
    98  }
    99  
   100  // Merge combines the markdown documents (files ending in ".md") in the tree rooted
   101  // at fs into a single document.
   102  // The blocks of the documents are concatenated in lexicographic order by filename.
   103  // Heading with no content are removed.
   104  // The link keys must be unique, and are combined into a single map.
   105  //
   106  // Files in the "minor changes" directory (the unique directory matching the glob
   107  // "*stdlib/*minor") are named after the package to which they refer, and will have
   108  // the package heading inserted automatically and links to other standard library
   109  // symbols expanded automatically. For example, if a file *stdlib/minor/bytes/f.md
   110  // contains the text
   111  //
   112  //	[Reader] implements [io.Reader].
   113  //
   114  // then that will become
   115  //
   116  //	[Reader](/pkg/bytes#Reader) implements [io.Reader](/pkg/io#Reader).
   117  func Merge(fsys fs.FS) (*md.Document, error) {
   118  	filenames, err := sortedMarkdownFilenames(fsys)
   119  	if err != nil {
   120  		return nil, err
   121  	}
   122  	doc := &md.Document{Links: map[string]*md.Link{}}
   123  	var prevPkg string // previous stdlib package, if any
   124  	for _, filename := range filenames {
   125  		newdoc, err := parseMarkdownFile(fsys, filename)
   126  		if err != nil {
   127  			return nil, err
   128  		}
   129  		if len(newdoc.Blocks) == 0 {
   130  			continue
   131  		}
   132  		pkg := stdlibPackage(filename)
   133  		// Autolink Go symbols.
   134  		addSymbolLinks(newdoc, pkg)
   135  		if len(doc.Blocks) > 0 {
   136  			// If this is the first file of a new stdlib package under the "Minor changes
   137  			// to the library" section, insert a heading for the package.
   138  			if pkg != "" && pkg != prevPkg {
   139  				h := stdlibPackageHeading(pkg, lastBlock(doc).Pos().EndLine)
   140  				doc.Blocks = append(doc.Blocks, h)
   141  			}
   142  			prevPkg = pkg
   143  			// Put a blank line between the current and new blocks, so that the end
   144  			// of a file acts as a blank line.
   145  			lastLine := lastBlock(doc).Pos().EndLine
   146  			delta := lastLine + 2 - newdoc.Blocks[0].Pos().StartLine
   147  			for _, b := range newdoc.Blocks {
   148  				addLines(b, delta)
   149  			}
   150  		}
   151  		// Append non-empty blocks to the result document.
   152  		for _, b := range newdoc.Blocks {
   153  			if _, ok := b.(*md.Empty); !ok {
   154  				doc.Blocks = append(doc.Blocks, b)
   155  			}
   156  		}
   157  		// Merge link references.
   158  		for key, link := range newdoc.Links {
   159  			if doc.Links[key] != nil {
   160  				return nil, fmt.Errorf("duplicate link reference %q; second in %s", key, filename)
   161  			}
   162  			doc.Links[key] = link
   163  		}
   164  	}
   165  	// Remove headings with empty contents.
   166  	doc.Blocks = removeEmptySections(doc.Blocks)
   167  	if len(doc.Blocks) > 0 && len(doc.Links) > 0 {
   168  		// Add a blank line to separate the links.
   169  		lastPos := lastBlock(doc).Pos()
   170  		lastPos.StartLine += 2
   171  		lastPos.EndLine += 2
   172  		doc.Blocks = append(doc.Blocks, &md.Empty{Position: lastPos})
   173  	}
   174  	return doc, nil
   175  }
   176  
   177  // stdlibPackage returns the standard library package for the given filename.
   178  // If the filename does not represent a package, it returns the empty string.
   179  // A filename represents package P if it is in a directory matching the glob
   180  // "*stdlib/*minor/P".
   181  func stdlibPackage(filename string) string {
   182  	dir, rest, _ := strings.Cut(filename, "/")
   183  	if !strings.HasSuffix(dir, "stdlib") {
   184  		return ""
   185  	}
   186  	dir, rest, _ = strings.Cut(rest, "/")
   187  	if !strings.HasSuffix(dir, "minor") {
   188  		return ""
   189  	}
   190  	pkg := path.Dir(rest)
   191  	if pkg == "." {
   192  		return ""
   193  	}
   194  	return pkg
   195  }
   196  
   197  func stdlibPackageHeading(pkg string, lastLine int) *md.Heading {
   198  	line := lastLine + 2
   199  	pos := md.Position{StartLine: line, EndLine: line}
   200  	return &md.Heading{
   201  		Position: pos,
   202  		Level:    4,
   203  		Text: &md.Text{
   204  			Position: pos,
   205  			Inline: []md.Inline{
   206  				&md.Link{
   207  					Inner: []md.Inline{&md.Code{Text: pkg}},
   208  					URL:   "/pkg/" + pkg + "/",
   209  				},
   210  			},
   211  		},
   212  	}
   213  }
   214  
   215  // removeEmptySections removes headings with no content. A heading has no content
   216  // if there are no blocks between it and the next heading at the same level, or the
   217  // end of the document.
   218  func removeEmptySections(bs []md.Block) []md.Block {
   219  	res := bs[:0]
   220  	delta := 0 // number of lines by which to adjust positions
   221  
   222  	// Remove preceding headings at same or higher level; they are empty.
   223  	rem := func(level int) {
   224  		for len(res) > 0 {
   225  			last := res[len(res)-1]
   226  			if lh, ok := last.(*md.Heading); ok && lh.Level >= level {
   227  				res = res[:len(res)-1]
   228  				// Adjust subsequent block positions by the size of this block
   229  				// plus 1 for the blank line between headings.
   230  				delta += lh.EndLine - lh.StartLine + 2
   231  			} else {
   232  				break
   233  			}
   234  		}
   235  	}
   236  
   237  	for _, b := range bs {
   238  		if h, ok := b.(*md.Heading); ok {
   239  			rem(h.Level)
   240  		}
   241  		addLines(b, -delta)
   242  		res = append(res, b)
   243  	}
   244  	// Remove empty headings at the end of the document.
   245  	rem(1)
   246  	return res
   247  }
   248  
   249  func sortedMarkdownFilenames(fsys fs.FS) ([]string, error) {
   250  	var filenames []string
   251  	err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
   252  		if err != nil {
   253  			return err
   254  		}
   255  		if !d.IsDir() && strings.HasSuffix(path, ".md") {
   256  			filenames = append(filenames, path)
   257  		}
   258  		return nil
   259  	})
   260  	if err != nil {
   261  		return nil, err
   262  	}
   263  	// '.' comes before '/', which comes before alphanumeric characters.
   264  	// So just sorting the list will put a filename like "net.md" before
   265  	// the directory "net". That is what we want.
   266  	slices.Sort(filenames)
   267  	return filenames, nil
   268  }
   269  
   270  // lastBlock returns the last block in the document.
   271  // It panics if the document has no blocks.
   272  func lastBlock(doc *md.Document) md.Block {
   273  	return doc.Blocks[len(doc.Blocks)-1]
   274  }
   275  
   276  // addLines adds n lines to the position of b.
   277  // n can be negative.
   278  func addLines(b md.Block, n int) {
   279  	pos := position(b)
   280  	pos.StartLine += n
   281  	pos.EndLine += n
   282  }
   283  
   284  func position(b md.Block) *md.Position {
   285  	switch b := b.(type) {
   286  	case *md.Heading:
   287  		return &b.Position
   288  	case *md.Text:
   289  		return &b.Position
   290  	case *md.CodeBlock:
   291  		return &b.Position
   292  	case *md.HTMLBlock:
   293  		return &b.Position
   294  	case *md.List:
   295  		return &b.Position
   296  	case *md.Item:
   297  		return &b.Position
   298  	case *md.Empty:
   299  		return &b.Position
   300  	case *md.Paragraph:
   301  		return &b.Position
   302  	case *md.Quote:
   303  		return &b.Position
   304  	case *md.ThematicBreak:
   305  		return &b.Position
   306  	default:
   307  		panic(fmt.Sprintf("unknown block type %T", b))
   308  	}
   309  }
   310  
   311  func parseMarkdownFile(fsys fs.FS, path string) (*md.Document, error) {
   312  	f, err := fsys.Open(path)
   313  	if err != nil {
   314  		return nil, err
   315  	}
   316  	defer f.Close()
   317  	data, err := io.ReadAll(f)
   318  	if err != nil {
   319  		return nil, err
   320  	}
   321  	in := string(data)
   322  	doc := NewParser().Parse(in)
   323  	return doc, nil
   324  }
   325  
   326  // An APIFeature is a symbol mentioned in an API file,
   327  // like the ones in the main go repo in the api directory.
   328  type APIFeature struct {
   329  	Package string // package that the feature is in
   330  	Build   string // build that the symbol is relevant for (e.g. GOOS, GOARCH)
   331  	Feature string // everything about the feature other than the package
   332  	Issue   int    // the issue that introduced the feature, or 0 if none
   333  }
   334  
   335  // This regexp has four capturing groups: package, build, feature and issue.
   336  var apiFileLineRegexp = regexp.MustCompile(`^pkg ([^ \t]+)[ \t]*(\([^)]+\))?, ([^#]*)(#\d+)?$`)
   337  
   338  // parseAPIFile parses a file in the api format and returns a list of the file's features.
   339  // A feature is represented by a single line that looks like
   340  //
   341  //	pkg PKG (BUILD) FEATURE #ISSUE
   342  //
   343  // where the BUILD and ISSUE may be absent.
   344  func parseAPIFile(fsys fs.FS, filename string) ([]APIFeature, error) {
   345  	f, err := fsys.Open(filename)
   346  	if err != nil {
   347  		return nil, err
   348  	}
   349  	defer f.Close()
   350  	var features []APIFeature
   351  	scan := bufio.NewScanner(f)
   352  	for scan.Scan() {
   353  		line := strings.TrimSpace(scan.Text())
   354  		if line == "" || line[0] == '#' {
   355  			continue
   356  		}
   357  		matches := apiFileLineRegexp.FindStringSubmatch(line)
   358  		if len(matches) == 0 {
   359  			return nil, fmt.Errorf("%s: malformed line %q", filename, line)
   360  		}
   361  		if len(matches) != 5 {
   362  			return nil, fmt.Errorf("wrong number of matches for line %q", line)
   363  		}
   364  		f := APIFeature{
   365  			Package: matches[1],
   366  			Build:   matches[2],
   367  			Feature: strings.TrimSpace(matches[3]),
   368  		}
   369  		if issue := matches[4]; issue != "" {
   370  			var err error
   371  			f.Issue, err = strconv.Atoi(issue[1:]) // skip leading '#'
   372  			if err != nil {
   373  				return nil, err
   374  			}
   375  		}
   376  		features = append(features, f)
   377  	}
   378  	if scan.Err() != nil {
   379  		return nil, scan.Err()
   380  	}
   381  	return features, nil
   382  }
   383  
   384  // GroupAPIFeaturesByFile returns a map of the given features keyed by
   385  // the doc filename that they are associated with.
   386  // A feature with package P and issue N should be documented in the file
   387  // "P/N.md".
   388  func GroupAPIFeaturesByFile(fs []APIFeature) (map[string][]APIFeature, error) {
   389  	m := map[string][]APIFeature{}
   390  	for _, f := range fs {
   391  		if f.Issue == 0 {
   392  			return nil, fmt.Errorf("%+v: zero issue", f)
   393  		}
   394  		filename := fmt.Sprintf("%s/%d.md", f.Package, f.Issue)
   395  		m[filename] = append(m[filename], f)
   396  	}
   397  	return m, nil
   398  }
   399  
   400  // CheckAPIFile reads the api file at filename in apiFS, and checks the corresponding
   401  // release-note files under docFS. It checks that the files exist and that they have
   402  // some minimal content (see [CheckFragment]).
   403  // The docRoot argument is the path from the repo or project root to the root of docFS.
   404  // It is used only for error messages.
   405  func CheckAPIFile(apiFS fs.FS, filename string, docFS fs.FS, docRoot string) error {
   406  	features, err := parseAPIFile(apiFS, filename)
   407  	if err != nil {
   408  		return err
   409  	}
   410  	byFile, err := GroupAPIFeaturesByFile(features)
   411  	if err != nil {
   412  		return err
   413  	}
   414  	var filenames []string
   415  	for fn := range byFile {
   416  		filenames = append(filenames, fn)
   417  	}
   418  	slices.Sort(filenames)
   419  	mcDir, err := minorChangesDir(docFS)
   420  	if err != nil {
   421  		return err
   422  	}
   423  	var errs []error
   424  	for _, fn := range filenames {
   425  		// Use path.Join for consistency with io/fs pathnames.
   426  		fn = path.Join(mcDir, fn)
   427  		// TODO(jba): check that the file mentions each feature?
   428  		if err := checkFragmentFile(docFS, fn); err != nil {
   429  			errs = append(errs, fmt.Errorf("%s: %v\nSee doc/README.md for more information.", path.Join(docRoot, fn), err))
   430  		}
   431  	}
   432  	return errors.Join(errs...)
   433  }
   434  
   435  // minorChangesDir returns the unique directory in docFS that corresponds to the
   436  // "Minor changes to the standard library" section of the release notes.
   437  func minorChangesDir(docFS fs.FS) (string, error) {
   438  	dirs, err := fs.Glob(docFS, "*stdlib/*minor")
   439  	if err != nil {
   440  		return "", err
   441  	}
   442  	var bad string
   443  	if len(dirs) == 0 {
   444  		bad = "No"
   445  	} else if len(dirs) > 1 {
   446  		bad = "More than one"
   447  	}
   448  	if bad != "" {
   449  		return "", fmt.Errorf("%s directory matches *stdlib/*minor.\nThis shouldn't happen; please file a bug at https://go.dev/issues/new.",
   450  			bad)
   451  	}
   452  	return dirs[0], nil
   453  }
   454  
   455  func checkFragmentFile(fsys fs.FS, filename string) error {
   456  	f, err := fsys.Open(filename)
   457  	if err != nil {
   458  		if errors.Is(err, fs.ErrNotExist) {
   459  			err = errors.New("File does not exist. Every API change must have a corresponding release note file.")
   460  		}
   461  		return err
   462  	}
   463  	defer f.Close()
   464  	data, err := io.ReadAll(f)
   465  	return CheckFragment(string(data))
   466  }
   467
View as plain text