formdata.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package multipart
     6  
     7  import (
     8  	"bytes"
     9  	"errors"
    10  	"internal/godebug"
    11  	"io"
    12  	"math"
    13  	"net/textproto"
    14  	"os"
    15  	"strconv"
    16  )
    17  
    18  // ErrMessageTooLarge is returned by ReadForm if the message form
    19  // data is too large to be processed.
    20  var ErrMessageTooLarge = errors.New("multipart: message too large")
    21  
    22  // TODO(adg,bradfitz): find a way to unify the DoS-prevention strategy here
    23  // with that of the http package's ParseForm.
    24  
    25  // ReadForm parses an entire multipart message whose parts have
    26  // a Content-Disposition of "form-data".
    27  // It stores up to maxMemory bytes + 10MB (reserved for non-file parts)
    28  // in memory. File parts which can't be stored in memory will be stored on
    29  // disk in temporary files.
    30  // It returns [ErrMessageTooLarge] if all non-file parts can't be stored in
    31  // memory.
    32  func (r *Reader) ReadForm(maxMemory int64) (*Form, error) {
    33  	return r.readForm(maxMemory)
    34  }
    35  
    36  var (
    37  	multipartfiles    = godebug.New("#multipartfiles") // TODO: document and remove #
    38  	multipartmaxparts = godebug.New("multipartmaxparts")
    39  )
    40  
    41  func (r *Reader) readForm(maxMemory int64) (_ *Form, err error) {
    42  	form := &Form{make(map[string][]string), make(map[string][]*FileHeader)}
    43  	var (
    44  		file    *os.File
    45  		fileOff int64
    46  	)
    47  	numDiskFiles := 0
    48  	combineFiles := true
    49  	if multipartfiles.Value() == "distinct" {
    50  		combineFiles = false
    51  		// multipartfiles.IncNonDefault() // TODO: uncomment after documenting
    52  	}
    53  	maxParts := 1000
    54  	if s := multipartmaxparts.Value(); s != "" {
    55  		if v, err := strconv.Atoi(s); err == nil && v >= 0 {
    56  			maxParts = v
    57  			multipartmaxparts.IncNonDefault()
    58  		}
    59  	}
    60  	maxHeaders := maxMIMEHeaders()
    61  
    62  	defer func() {
    63  		if file != nil {
    64  			if cerr := file.Close(); err == nil {
    65  				err = cerr
    66  			}
    67  		}
    68  		if combineFiles && numDiskFiles > 1 {
    69  			for _, fhs := range form.File {
    70  				for _, fh := range fhs {
    71  					fh.tmpshared = true
    72  				}
    73  			}
    74  		}
    75  		if err != nil {
    76  			form.RemoveAll()
    77  			if file != nil {
    78  				os.Remove(file.Name())
    79  			}
    80  		}
    81  	}()
    82  
    83  	// maxFileMemoryBytes is the maximum bytes of file data we will store in memory.
    84  	// Data past this limit is written to disk.
    85  	// This limit strictly applies to content, not metadata (filenames, MIME headers, etc.),
    86  	// since metadata is always stored in memory, not disk.
    87  	//
    88  	// maxMemoryBytes is the maximum bytes we will store in memory, including file content,
    89  	// non-file part values, metadata, and map entry overhead.
    90  	//
    91  	// We reserve an additional 10 MB in maxMemoryBytes for non-file data.
    92  	//
    93  	// The relationship between these parameters, as well as the overly-large and
    94  	// unconfigurable 10 MB added on to maxMemory, is unfortunate but difficult to change
    95  	// within the constraints of the API as documented.
    96  	maxFileMemoryBytes := maxMemory
    97  	if maxFileMemoryBytes == math.MaxInt64 {
    98  		maxFileMemoryBytes--
    99  	}
   100  	maxMemoryBytes := maxMemory + int64(10<<20)
   101  	if maxMemoryBytes <= 0 {
   102  		if maxMemory < 0 {
   103  			maxMemoryBytes = 0
   104  		} else {
   105  			maxMemoryBytes = math.MaxInt64
   106  		}
   107  	}
   108  	var copyBuf []byte
   109  	for {
   110  		p, err := r.nextPart(false, maxMemoryBytes, maxHeaders)
   111  		if err == io.EOF {
   112  			break
   113  		}
   114  		if err != nil {
   115  			return nil, err
   116  		}
   117  		if maxParts <= 0 {
   118  			return nil, ErrMessageTooLarge
   119  		}
   120  		maxParts--
   121  
   122  		name := p.FormName()
   123  		if name == "" {
   124  			continue
   125  		}
   126  		filename := p.FileName()
   127  
   128  		// Multiple values for the same key (one map entry, longer slice) are cheaper
   129  		// than the same number of values for different keys (many map entries), but
   130  		// using a consistent per-value cost for overhead is simpler.
   131  		const mapEntryOverhead = 200
   132  		maxMemoryBytes -= int64(len(name))
   133  		maxMemoryBytes -= mapEntryOverhead
   134  		if maxMemoryBytes < 0 {
   135  			// We can't actually take this path, since nextPart would already have
   136  			// rejected the MIME headers for being too large. Check anyway.
   137  			return nil, ErrMessageTooLarge
   138  		}
   139  
   140  		var b bytes.Buffer
   141  
   142  		if filename == "" {
   143  			// value, store as string in memory
   144  			n, err := io.CopyN(&b, p, maxMemoryBytes+1)
   145  			if err != nil && err != io.EOF {
   146  				return nil, err
   147  			}
   148  			maxMemoryBytes -= n
   149  			if maxMemoryBytes < 0 {
   150  				return nil, ErrMessageTooLarge
   151  			}
   152  			form.Value[name] = append(form.Value[name], b.String())
   153  			continue
   154  		}
   155  
   156  		// file, store in memory or on disk
   157  		const fileHeaderSize = 100
   158  		maxMemoryBytes -= mimeHeaderSize(p.Header)
   159  		maxMemoryBytes -= mapEntryOverhead
   160  		maxMemoryBytes -= fileHeaderSize
   161  		if maxMemoryBytes < 0 {
   162  			return nil, ErrMessageTooLarge
   163  		}
   164  		for _, v := range p.Header {
   165  			maxHeaders -= int64(len(v))
   166  		}
   167  		fh := &FileHeader{
   168  			Filename: filename,
   169  			Header:   p.Header,
   170  		}
   171  		n, err := io.CopyN(&b, p, maxFileMemoryBytes+1)
   172  		if err != nil && err != io.EOF {
   173  			return nil, err
   174  		}
   175  		if n > maxFileMemoryBytes {
   176  			if file == nil {
   177  				file, err = os.CreateTemp(r.tempDir, "multipart-")
   178  				if err != nil {
   179  					return nil, err
   180  				}
   181  			}
   182  			numDiskFiles++
   183  			if _, err := file.Write(b.Bytes()); err != nil {
   184  				return nil, err
   185  			}
   186  			if copyBuf == nil {
   187  				copyBuf = make([]byte, 32*1024) // same buffer size as io.Copy uses
   188  			}
   189  			// os.File.ReadFrom will allocate its own copy buffer if we let io.Copy use it.
   190  			type writerOnly struct{ io.Writer }
   191  			remainingSize, err := io.CopyBuffer(writerOnly{file}, p, copyBuf)
   192  			if err != nil {
   193  				return nil, err
   194  			}
   195  			fh.tmpfile = file.Name()
   196  			fh.Size = int64(b.Len()) + remainingSize
   197  			fh.tmpoff = fileOff
   198  			fileOff += fh.Size
   199  			if !combineFiles {
   200  				if err := file.Close(); err != nil {
   201  					return nil, err
   202  				}
   203  				file = nil
   204  			}
   205  		} else {
   206  			fh.content = b.Bytes()
   207  			fh.Size = int64(len(fh.content))
   208  			maxFileMemoryBytes -= n
   209  			maxMemoryBytes -= n
   210  		}
   211  		form.File[name] = append(form.File[name], fh)
   212  	}
   213  
   214  	return form, nil
   215  }
   216  
   217  func mimeHeaderSize(h textproto.MIMEHeader) (size int64) {
   218  	size = 400
   219  	for k, vs := range h {
   220  		size += int64(len(k))
   221  		size += 200 // map entry overhead
   222  		for _, v := range vs {
   223  			size += int64(len(v))
   224  		}
   225  	}
   226  	return size
   227  }
   228  
   229  // Form is a parsed multipart form.
   230  // Its File parts are stored either in memory or on disk,
   231  // and are accessible via the [*FileHeader]'s Open method.
   232  // Its Value parts are stored as strings.
   233  // Both are keyed by field name.
   234  type Form struct {
   235  	Value map[string][]string
   236  	File  map[string][]*FileHeader
   237  }
   238  
   239  // RemoveAll removes any temporary files associated with a [Form].
   240  func (f *Form) RemoveAll() error {
   241  	var err error
   242  	for _, fhs := range f.File {
   243  		for _, fh := range fhs {
   244  			if fh.tmpfile != "" {
   245  				e := os.Remove(fh.tmpfile)
   246  				if e != nil && !errors.Is(e, os.ErrNotExist) && err == nil {
   247  					err = e
   248  				}
   249  			}
   250  		}
   251  	}
   252  	return err
   253  }
   254  
   255  // A FileHeader describes a file part of a multipart request.
   256  type FileHeader struct {
   257  	Filename string
   258  	Header   textproto.MIMEHeader
   259  	Size     int64
   260  
   261  	content   []byte
   262  	tmpfile   string
   263  	tmpoff    int64
   264  	tmpshared bool
   265  }
   266  
   267  // Open opens and returns the [FileHeader]'s associated File.
   268  func (fh *FileHeader) Open() (File, error) {
   269  	if b := fh.content; b != nil {
   270  		r := io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b)))
   271  		return sectionReadCloser{r, nil}, nil
   272  	}
   273  	if fh.tmpshared {
   274  		f, err := os.Open(fh.tmpfile)
   275  		if err != nil {
   276  			return nil, err
   277  		}
   278  		r := io.NewSectionReader(f, fh.tmpoff, fh.Size)
   279  		return sectionReadCloser{r, f}, nil
   280  	}
   281  	return os.Open(fh.tmpfile)
   282  }
   283  
   284  // File is an interface to access the file part of a multipart message.
   285  // Its contents may be either stored in memory or on disk.
   286  // If stored on disk, the File's underlying concrete type will be an *os.File.
   287  type File interface {
   288  	io.Reader
   289  	io.ReaderAt
   290  	io.Seeker
   291  	io.Closer
   292  }
   293  
   294  // helper types to turn a []byte into a File
   295  
   296  type sectionReadCloser struct {
   297  	*io.SectionReader
   298  	io.Closer
   299  }
   300  
   301  func (rc sectionReadCloser) Close() error {
   302  	if rc.Closer != nil {
   303  		return rc.Closer.Close()
   304  	}
   305  	return nil
   306  }
   307
View as plain text