zero_copy_linux.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package os
     6  
     7  import (
     8  	"internal/poll"
     9  	"io"
    10  	"syscall"
    11  )
    12  
    13  var (
    14  	pollCopyFileRange = poll.CopyFileRange
    15  	pollSplice        = poll.Splice
    16  )
    17  
    18  func (f *File) writeTo(w io.Writer) (written int64, handled bool, err error) {
    19  	pfd, network := getPollFDAndNetwork(w)
    20  	// TODO(panjf2000): same as File.spliceToFile.
    21  	if pfd == nil || !pfd.IsStream || !isUnixOrTCP(string(network)) {
    22  		return
    23  	}
    24  
    25  	sc, err := f.SyscallConn()
    26  	if err != nil {
    27  		return
    28  	}
    29  
    30  	rerr := sc.Read(func(fd uintptr) (done bool) {
    31  		written, err, handled = poll.SendFile(pfd, int(fd), 0)
    32  		return true
    33  	})
    34  
    35  	if err == nil {
    36  		err = rerr
    37  	}
    38  
    39  	return written, handled, wrapSyscallError("sendfile", err)
    40  }
    41  
    42  func (f *File) readFrom(r io.Reader) (written int64, handled bool, err error) {
    43  	// Neither copy_file_range(2)/sendfile(2) nor splice(2) supports destinations opened with
    44  	// O_APPEND, so don't bother to try zero-copy with these system calls.
    45  	//
    46  	// Visit https://man7.org/linux/man-pages/man2/copy_file_range.2.html#ERRORS and
    47  	// https://man7.org/linux/man-pages/man2/sendfile.2.html#ERRORS and
    48  	// https://man7.org/linux/man-pages/man2/splice.2.html#ERRORS for details.
    49  	if f.appendMode {
    50  		return 0, false, nil
    51  	}
    52  
    53  	written, handled, err = f.copyFile(r)
    54  	if handled {
    55  		return
    56  	}
    57  	return f.spliceToFile(r)
    58  }
    59  
    60  func (f *File) spliceToFile(r io.Reader) (written int64, handled bool, err error) {
    61  	var (
    62  		remain int64
    63  		lr     *io.LimitedReader
    64  	)
    65  	if lr, r, remain = tryLimitedReader(r); remain <= 0 {
    66  		return 0, true, nil
    67  	}
    68  
    69  	pfd, _ := getPollFDAndNetwork(r)
    70  	// TODO(panjf2000): run some tests to see if we should unlock the non-streams for splice.
    71  	// Streams benefit the most from the splice(2), non-streams are not even supported in old kernels
    72  	// where splice(2) will just return EINVAL; newer kernels support non-streams like UDP, but I really
    73  	// doubt that splice(2) could help non-streams, cuz they usually send small frames respectively
    74  	// and one splice call would result in one frame.
    75  	// splice(2) is suitable for large data but the generation of fragments defeats its edge here.
    76  	// Therefore, don't bother to try splice if the r is not a streaming descriptor.
    77  	if pfd == nil || !pfd.IsStream {
    78  		return
    79  	}
    80  
    81  	written, handled, err = pollSplice(&f.pfd, pfd, remain)
    82  
    83  	if lr != nil {
    84  		lr.N = remain - written
    85  	}
    86  
    87  	return written, handled, wrapSyscallError("splice", err)
    88  }
    89  
    90  func (f *File) copyFile(r io.Reader) (written int64, handled bool, err error) {
    91  	var (
    92  		remain int64
    93  		lr     *io.LimitedReader
    94  	)
    95  	if lr, r, remain = tryLimitedReader(r); remain <= 0 {
    96  		return 0, true, nil
    97  	}
    98  
    99  	var src *File
   100  	switch v := r.(type) {
   101  	case *File:
   102  		src = v
   103  	case fileWithoutWriteTo:
   104  		src = v.File
   105  	default:
   106  		return 0, false, nil
   107  	}
   108  
   109  	if src.checkValid("ReadFrom") != nil {
   110  		// Avoid returning the error as we report handled as false,
   111  		// leave further error handling as the responsibility of the caller.
   112  		return 0, false, nil
   113  	}
   114  
   115  	written, handled, err = pollCopyFileRange(&f.pfd, &src.pfd, remain)
   116  	if lr != nil {
   117  		lr.N -= written
   118  	}
   119  
   120  	if handled {
   121  		return written, handled, wrapSyscallError("copy_file_range", err)
   122  	}
   123  
   124  	// If fd_in and fd_out refer to the same file and the source and target ranges overlap,
   125  	// copy_file_range(2) just returns EINVAL error. poll.CopyFileRange will ignore that
   126  	// error and act like it didn't call copy_file_range(2). Then the caller will fall back
   127  	// to generic copy, which results in doubling the content in the file.
   128  	// By contrast, sendfile(2) allows this kind of overlapping and works like a memmove,
   129  	// in this case the file content will remain the same after copying, which is not what we want.
   130  	// Thus, we just bail out here and leave it to generic copy when it's a file copying itself.
   131  	if f.pfd.Sysfd == src.pfd.Sysfd {
   132  		return 0, false, nil
   133  	}
   134  
   135  	sc, err := src.SyscallConn()
   136  	if err != nil {
   137  		return
   138  	}
   139  
   140  	// We can employ sendfile(2) when copy_file_range(2) fails to handle the copy.
   141  	// sendfile(2) enabled file-to-file copying since Linux 2.6.33 and Go requires
   142  	// Linux 3.2 or later, so we're good to go.
   143  	// Check out https://man7.org/linux/man-pages/man2/sendfile.2.html#DESCRIPTION for more details.
   144  	rerr := sc.Read(func(fd uintptr) bool {
   145  		written, err, handled = poll.SendFile(&f.pfd, int(fd), remain)
   146  		return true
   147  	})
   148  	if lr != nil {
   149  		lr.N -= written
   150  	}
   151  
   152  	if err == nil {
   153  		err = rerr
   154  	}
   155  
   156  	return written, handled, wrapSyscallError("sendfile", err)
   157  }
   158  
   159  // getPollFDAndNetwork tries to get the poll.FD and network type from the given interface
   160  // by expecting the underlying type of i to be the implementation of syscall.Conn
   161  // that contains a *net.rawConn.
   162  func getPollFDAndNetwork(i any) (*poll.FD, poll.String) {
   163  	sc, ok := i.(syscall.Conn)
   164  	if !ok {
   165  		return nil, ""
   166  	}
   167  	rc, err := sc.SyscallConn()
   168  	if err != nil {
   169  		return nil, ""
   170  	}
   171  	irc, ok := rc.(interface {
   172  		PollFD() *poll.FD
   173  		Network() poll.String
   174  	})
   175  	if !ok {
   176  		return nil, ""
   177  	}
   178  	return irc.PollFD(), irc.Network()
   179  }
   180  
   181  func isUnixOrTCP(network string) bool {
   182  	switch network {
   183  	case "tcp", "tcp4", "tcp6", "unix":
   184  		return true
   185  	default:
   186  		return false
   187  	}
   188  }
   189
View as plain text