// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package binary implements simple translation between numbers and byte // sequences and encoding and decoding of varints. // // Numbers are translated by reading and writing fixed-size values. // A fixed-size value is either a fixed-size arithmetic // type (bool, int8, uint8, int16, float32, complex64, ...) // or an array or struct containing only fixed-size values. // // The varint functions encode and decode single integer values using // a variable-length encoding; smaller values require fewer bytes. // For a specification, see // https://developers.google.com/protocol-buffers/docs/encoding. // // This package favors simplicity over efficiency. Clients that require // high-performance serialization, especially for large data structures, // should look at more advanced solutions such as the [encoding/gob] // package or [google.golang.org/protobuf] for protocol buffers. package binary import ( "errors" "io" "math" "reflect" "slices" "sync" ) var errBufferTooSmall = errors.New("buffer too small") // A ByteOrder specifies how to convert byte slices into // 16-, 32-, or 64-bit unsigned integers. // // It is implemented by [LittleEndian], [BigEndian], and [NativeEndian]. type ByteOrder interface { Uint16([]byte) uint16 Uint32([]byte) uint32 Uint64([]byte) uint64 PutUint16([]byte, uint16) PutUint32([]byte, uint32) PutUint64([]byte, uint64) String() string } // AppendByteOrder specifies how to append 16-, 32-, or 64-bit unsigned integers // into a byte slice. // // It is implemented by [LittleEndian], [BigEndian], and [NativeEndian]. type AppendByteOrder interface { AppendUint16([]byte, uint16) []byte AppendUint32([]byte, uint32) []byte AppendUint64([]byte, uint64) []byte String() string } // LittleEndian is the little-endian implementation of [ByteOrder] and [AppendByteOrder]. var LittleEndian littleEndian // BigEndian is the big-endian implementation of [ByteOrder] and [AppendByteOrder]. var BigEndian bigEndian type littleEndian struct{} func (littleEndian) Uint16(b []byte) uint16 { _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 return uint16(b[0]) | uint16(b[1])<<8 } func (littleEndian) PutUint16(b []byte, v uint16) { _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) } func (littleEndian) AppendUint16(b []byte, v uint16) []byte { return append(b, byte(v), byte(v>>8), ) } func (littleEndian) Uint32(b []byte) uint32 { _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 } func (littleEndian) PutUint32(b []byte, v uint32) { _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) b[2] = byte(v >> 16) b[3] = byte(v >> 24) } func (littleEndian) AppendUint32(b []byte, v uint32) []byte { return append(b, byte(v), byte(v>>8), byte(v>>16), byte(v>>24), ) } func (littleEndian) Uint64(b []byte) uint64 { _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 } func (littleEndian) PutUint64(b []byte, v uint64) { _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) b[2] = byte(v >> 16) b[3] = byte(v >> 24) b[4] = byte(v >> 32) b[5] = byte(v >> 40) b[6] = byte(v >> 48) b[7] = byte(v >> 56) } func (littleEndian) AppendUint64(b []byte, v uint64) []byte { return append(b, byte(v), byte(v>>8), byte(v>>16), byte(v>>24), byte(v>>32), byte(v>>40), byte(v>>48), byte(v>>56), ) } func (littleEndian) String() string { return "LittleEndian" } func (littleEndian) GoString() string { return "binary.LittleEndian" } type bigEndian struct{} func (bigEndian) Uint16(b []byte) uint16 { _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 return uint16(b[1]) | uint16(b[0])<<8 } func (bigEndian) PutUint16(b []byte, v uint16) { _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 8) b[1] = byte(v) } func (bigEndian) AppendUint16(b []byte, v uint16) []byte { return append(b, byte(v>>8), byte(v), ) } func (bigEndian) Uint32(b []byte) uint32 { _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 } func (bigEndian) PutUint32(b []byte, v uint32) { _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 24) b[1] = byte(v >> 16) b[2] = byte(v >> 8) b[3] = byte(v) } func (bigEndian) AppendUint32(b []byte, v uint32) []byte { return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v), ) } func (bigEndian) Uint64(b []byte) uint64 { _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 } func (bigEndian) PutUint64(b []byte, v uint64) { _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 56) b[1] = byte(v >> 48) b[2] = byte(v >> 40) b[3] = byte(v >> 32) b[4] = byte(v >> 24) b[5] = byte(v >> 16) b[6] = byte(v >> 8) b[7] = byte(v) } func (bigEndian) AppendUint64(b []byte, v uint64) []byte { return append(b, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), byte(v), ) } func (bigEndian) String() string { return "BigEndian" } func (bigEndian) GoString() string { return "binary.BigEndian" } func (nativeEndian) String() string { return "NativeEndian" } func (nativeEndian) GoString() string { return "binary.NativeEndian" } // Read reads structured binary data from r into data. // Data must be a pointer to a fixed-size value or a slice // of fixed-size values. // Bytes read from r are decoded using the specified byte order // and written to successive fields of the data. // When decoding boolean values, a zero byte is decoded as false, and // any other non-zero byte is decoded as true. // When reading into structs, the field data for fields with // blank (_) field names is skipped; i.e., blank field names // may be used for padding. // When reading into a struct, all non-blank fields must be exported // or Read may panic. // // The error is [io.EOF] only if no bytes were read. // If an [io.EOF] happens after reading some but not all the bytes, // Read returns [io.ErrUnexpectedEOF]. func Read(r io.Reader, order ByteOrder, data any) error { // Fast path for basic types and slices. if n, _ := intDataSize(data); n != 0 { bs := make([]byte, n) if _, err := io.ReadFull(r, bs); err != nil { return err } if decodeFast(bs, order, data) { return nil } } // Fallback to reflect-based decoding. v := reflect.ValueOf(data) size := -1 switch v.Kind() { case reflect.Pointer: v = v.Elem() size = dataSize(v) case reflect.Slice: size = dataSize(v) } if size < 0 { return errors.New("binary.Read: invalid type " + reflect.TypeOf(data).String()) } d := &decoder{order: order, buf: make([]byte, size)} if _, err := io.ReadFull(r, d.buf); err != nil { return err } d.value(v) return nil } // Decode decodes binary data from buf into data according to // the given byte order. // It returns an error if buf is too small, otherwise the number of // bytes consumed from buf. func Decode(buf []byte, order ByteOrder, data any) (int, error) { if n, _ := intDataSize(data); n != 0 { if len(buf) < n { return 0, errBufferTooSmall } if decodeFast(buf, order, data) { return n, nil } } // Fallback to reflect-based decoding. v := reflect.ValueOf(data) size := -1 switch v.Kind() { case reflect.Pointer: v = v.Elem() size = dataSize(v) case reflect.Slice: size = dataSize(v) } if size < 0 { return 0, errors.New("binary.Decode: invalid type " + reflect.TypeOf(data).String()) } if len(buf) < size { return 0, errBufferTooSmall } d := &decoder{order: order, buf: buf[:size]} d.value(v) return size, nil } func decodeFast(bs []byte, order ByteOrder, data any) bool { switch data := data.(type) { case *bool: *data = bs[0] != 0 case *int8: *data = int8(bs[0]) case *uint8: *data = bs[0] case *int16: *data = int16(order.Uint16(bs)) case *uint16: *data = order.Uint16(bs) case *int32: *data = int32(order.Uint32(bs)) case *uint32: *data = order.Uint32(bs) case *int64: *data = int64(order.Uint64(bs)) case *uint64: *data = order.Uint64(bs) case *float32: *data = math.Float32frombits(order.Uint32(bs)) case *float64: *data = math.Float64frombits(order.Uint64(bs)) case []bool: for i, x := range bs { // Easier to loop over the input for 8-bit values. data[i] = x != 0 } case []int8: for i, x := range bs { data[i] = int8(x) } case []uint8: copy(data, bs) case []int16: for i := range data { data[i] = int16(order.Uint16(bs[2*i:])) } case []uint16: for i := range data { data[i] = order.Uint16(bs[2*i:]) } case []int32: for i := range data { data[i] = int32(order.Uint32(bs[4*i:])) } case []uint32: for i := range data { data[i] = order.Uint32(bs[4*i:]) } case []int64: for i := range data { data[i] = int64(order.Uint64(bs[8*i:])) } case []uint64: for i := range data { data[i] = order.Uint64(bs[8*i:]) } case []float32: for i := range data { data[i] = math.Float32frombits(order.Uint32(bs[4*i:])) } case []float64: for i := range data { data[i] = math.Float64frombits(order.Uint64(bs[8*i:])) } default: return false } return true } // Write writes the binary representation of data into w. // Data must be a fixed-size value or a slice of fixed-size // values, or a pointer to such data. // Boolean values encode as one byte: 1 for true, and 0 for false. // Bytes written to w are encoded using the specified byte order // and read from successive fields of the data. // When writing structs, zero values are written for fields // with blank (_) field names. func Write(w io.Writer, order ByteOrder, data any) error { // Fast path for basic types and slices. if n, bs := intDataSize(data); n != 0 { if bs == nil { bs = make([]byte, n) encodeFast(bs, order, data) } _, err := w.Write(bs) return err } // Fallback to reflect-based encoding. v := reflect.Indirect(reflect.ValueOf(data)) size := dataSize(v) if size < 0 { return errors.New("binary.Write: some values are not fixed-sized in type " + reflect.TypeOf(data).String()) } buf := make([]byte, size) e := &encoder{order: order, buf: buf} e.value(v) _, err := w.Write(buf) return err } // Encode encodes the binary representation of data into buf according to // the given byte order. // It returns an error if buf is too small, otherwise the number of // bytes written into buf. func Encode(buf []byte, order ByteOrder, data any) (int, error) { // Fast path for basic types and slices. if n, _ := intDataSize(data); n != 0 { if len(buf) < n { return 0, errBufferTooSmall } encodeFast(buf, order, data) return n, nil } // Fallback to reflect-based encoding. v := reflect.Indirect(reflect.ValueOf(data)) size := dataSize(v) if size < 0 { return 0, errors.New("binary.Encode: some values are not fixed-sized in type " + reflect.TypeOf(data).String()) } if len(buf) < size { return 0, errBufferTooSmall } e := &encoder{order: order, buf: buf} e.value(v) return size, nil } // Append appends the binary representation of data to buf. // buf may be nil, in which case a new buffer will be allocated. // See [Write] on which data are acceptable. // It returns the (possibily extended) buffer containing data or an error. func Append(buf []byte, order ByteOrder, data any) ([]byte, error) { // Fast path for basic types and slices. if n, _ := intDataSize(data); n != 0 { buf, pos := ensure(buf, n) encodeFast(pos, order, data) return buf, nil } // Fallback to reflect-based encoding. v := reflect.Indirect(reflect.ValueOf(data)) size := dataSize(v) if size < 0 { return nil, errors.New("binary.Append: some values are not fixed-sized in type " + reflect.TypeOf(data).String()) } buf, pos := ensure(buf, size) e := &encoder{order: order, buf: pos} e.value(v) return buf, nil } func encodeFast(bs []byte, order ByteOrder, data any) { switch v := data.(type) { case *bool: if *v { bs[0] = 1 } else { bs[0] = 0 } case bool: if v { bs[0] = 1 } else { bs[0] = 0 } case []bool: for i, x := range v { if x { bs[i] = 1 } else { bs[i] = 0 } } case *int8: bs[0] = byte(*v) case int8: bs[0] = byte(v) case []int8: for i, x := range v { bs[i] = byte(x) } case *uint8: bs[0] = *v case uint8: bs[0] = v case []uint8: copy(bs, v) case *int16: order.PutUint16(bs, uint16(*v)) case int16: order.PutUint16(bs, uint16(v)) case []int16: for i, x := range v { order.PutUint16(bs[2*i:], uint16(x)) } case *uint16: order.PutUint16(bs, *v) case uint16: order.PutUint16(bs, v) case []uint16: for i, x := range v { order.PutUint16(bs[2*i:], x) } case *int32: order.PutUint32(bs, uint32(*v)) case int32: order.PutUint32(bs, uint32(v)) case []int32: for i, x := range v { order.PutUint32(bs[4*i:], uint32(x)) } case *uint32: order.PutUint32(bs, *v) case uint32: order.PutUint32(bs, v) case []uint32: for i, x := range v { order.PutUint32(bs[4*i:], x) } case *int64: order.PutUint64(bs, uint64(*v)) case int64: order.PutUint64(bs, uint64(v)) case []int64: for i, x := range v { order.PutUint64(bs[8*i:], uint64(x)) } case *uint64: order.PutUint64(bs, *v) case uint64: order.PutUint64(bs, v) case []uint64: for i, x := range v { order.PutUint64(bs[8*i:], x) } case *float32: order.PutUint32(bs, math.Float32bits(*v)) case float32: order.PutUint32(bs, math.Float32bits(v)) case []float32: for i, x := range v { order.PutUint32(bs[4*i:], math.Float32bits(x)) } case *float64: order.PutUint64(bs, math.Float64bits(*v)) case float64: order.PutUint64(bs, math.Float64bits(v)) case []float64: for i, x := range v { order.PutUint64(bs[8*i:], math.Float64bits(x)) } } } // Size returns how many bytes [Write] would generate to encode the value v, which // must be a fixed-size value or a slice of fixed-size values, or a pointer to such data. // If v is neither of these, Size returns -1. func Size(v any) int { switch data := v.(type) { case bool, int8, uint8: return 1 case *bool: if data == nil { return -1 } return 1 case *int8: if data == nil { return -1 } return 1 case *uint8: if data == nil { return -1 } return 1 case []bool: return len(data) case []int8: return len(data) case []uint8: return len(data) case int16, uint16: return 2 case *int16: if data == nil { return -1 } return 2 case *uint16: if data == nil { return -1 } return 2 case []int16: return 2 * len(data) case []uint16: return 2 * len(data) case int32, uint32: return 4 case *int32: if data == nil { return -1 } return 4 case *uint32: if data == nil { return -1 } return 4 case []int32: return 4 * len(data) case []uint32: return 4 * len(data) case int64, uint64: return 8 case *int64: if data == nil { return -1 } return 8 case *uint64: if data == nil { return -1 } return 8 case []int64: return 8 * len(data) case []uint64: return 8 * len(data) case float32: return 4 case *float32: if data == nil { return -1 } return 4 case float64: return 8 case *float64: if data == nil { return -1 } return 8 case []float32: return 4 * len(data) case []float64: return 8 * len(data) } return dataSize(reflect.Indirect(reflect.ValueOf(v))) } var structSize sync.Map // map[reflect.Type]int // dataSize returns the number of bytes the actual data represented by v occupies in memory. // For compound structures, it sums the sizes of the elements. Thus, for instance, for a slice // it returns the length of the slice times the element size and does not count the memory // occupied by the header. If the type of v is not acceptable, dataSize returns -1. func dataSize(v reflect.Value) int { switch v.Kind() { case reflect.Slice, reflect.Array: t := v.Type().Elem() if size, ok := structSize.Load(t); ok { return size.(int) * v.Len() } size := sizeof(t) if size >= 0 { if t.Kind() == reflect.Struct { structSize.Store(t, size) } return size * v.Len() } case reflect.Struct: t := v.Type() if size, ok := structSize.Load(t); ok { return size.(int) } size := sizeof(t) structSize.Store(t, size) return size default: if v.IsValid() { return sizeof(v.Type()) } } return -1 } // sizeof returns the size >= 0 of variables for the given type or -1 if the type is not acceptable. func sizeof(t reflect.Type) int { switch t.Kind() { case reflect.Array: if s := sizeof(t.Elem()); s >= 0 { return s * t.Len() } case reflect.Struct: sum := 0 for i, n := 0, t.NumField(); i < n; i++ { s := sizeof(t.Field(i).Type) if s < 0 { return -1 } sum += s } return sum case reflect.Bool, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: return int(t.Size()) } return -1 } type coder struct { order ByteOrder buf []byte offset int } type decoder coder type encoder coder func (d *decoder) bool() bool { x := d.buf[d.offset] d.offset++ return x != 0 } func (e *encoder) bool(x bool) { if x { e.buf[e.offset] = 1 } else { e.buf[e.offset] = 0 } e.offset++ } func (d *decoder) uint8() uint8 { x := d.buf[d.offset] d.offset++ return x } func (e *encoder) uint8(x uint8) { e.buf[e.offset] = x e.offset++ } func (d *decoder) uint16() uint16 { x := d.order.Uint16(d.buf[d.offset : d.offset+2]) d.offset += 2 return x } func (e *encoder) uint16(x uint16) { e.order.PutUint16(e.buf[e.offset:e.offset+2], x) e.offset += 2 } func (d *decoder) uint32() uint32 { x := d.order.Uint32(d.buf[d.offset : d.offset+4]) d.offset += 4 return x } func (e *encoder) uint32(x uint32) { e.order.PutUint32(e.buf[e.offset:e.offset+4], x) e.offset += 4 } func (d *decoder) uint64() uint64 { x := d.order.Uint64(d.buf[d.offset : d.offset+8]) d.offset += 8 return x } func (e *encoder) uint64(x uint64) { e.order.PutUint64(e.buf[e.offset:e.offset+8], x) e.offset += 8 } func (d *decoder) int8() int8 { return int8(d.uint8()) } func (e *encoder) int8(x int8) { e.uint8(uint8(x)) } func (d *decoder) int16() int16 { return int16(d.uint16()) } func (e *encoder) int16(x int16) { e.uint16(uint16(x)) } func (d *decoder) int32() int32 { return int32(d.uint32()) } func (e *encoder) int32(x int32) { e.uint32(uint32(x)) } func (d *decoder) int64() int64 { return int64(d.uint64()) } func (e *encoder) int64(x int64) { e.uint64(uint64(x)) } func (d *decoder) value(v reflect.Value) { switch v.Kind() { case reflect.Array: l := v.Len() for i := 0; i < l; i++ { d.value(v.Index(i)) } case reflect.Struct: t := v.Type() l := v.NumField() for i := 0; i < l; i++ { // Note: Calling v.CanSet() below is an optimization. // It would be sufficient to check the field name, // but creating the StructField info for each field is // costly (run "go test -bench=ReadStruct" and compare // results when making changes to this code). if v := v.Field(i); v.CanSet() || t.Field(i).Name != "_" { d.value(v) } else { d.skip(v) } } case reflect.Slice: l := v.Len() for i := 0; i < l; i++ { d.value(v.Index(i)) } case reflect.Bool: v.SetBool(d.bool()) case reflect.Int8: v.SetInt(int64(d.int8())) case reflect.Int16: v.SetInt(int64(d.int16())) case reflect.Int32: v.SetInt(int64(d.int32())) case reflect.Int64: v.SetInt(d.int64()) case reflect.Uint8: v.SetUint(uint64(d.uint8())) case reflect.Uint16: v.SetUint(uint64(d.uint16())) case reflect.Uint32: v.SetUint(uint64(d.uint32())) case reflect.Uint64: v.SetUint(d.uint64()) case reflect.Float32: v.SetFloat(float64(math.Float32frombits(d.uint32()))) case reflect.Float64: v.SetFloat(math.Float64frombits(d.uint64())) case reflect.Complex64: v.SetComplex(complex( float64(math.Float32frombits(d.uint32())), float64(math.Float32frombits(d.uint32())), )) case reflect.Complex128: v.SetComplex(complex( math.Float64frombits(d.uint64()), math.Float64frombits(d.uint64()), )) } } func (e *encoder) value(v reflect.Value) { switch v.Kind() { case reflect.Array: l := v.Len() for i := 0; i < l; i++ { e.value(v.Index(i)) } case reflect.Struct: t := v.Type() l := v.NumField() for i := 0; i < l; i++ { // see comment for corresponding code in decoder.value() if v := v.Field(i); v.CanSet() || t.Field(i).Name != "_" { e.value(v) } else { e.skip(v) } } case reflect.Slice: l := v.Len() for i := 0; i < l; i++ { e.value(v.Index(i)) } case reflect.Bool: e.bool(v.Bool()) case reflect.Int8: e.int8(int8(v.Int())) case reflect.Int16: e.int16(int16(v.Int())) case reflect.Int32: e.int32(int32(v.Int())) case reflect.Int64: e.int64(v.Int()) case reflect.Uint8: e.uint8(uint8(v.Uint())) case reflect.Uint16: e.uint16(uint16(v.Uint())) case reflect.Uint32: e.uint32(uint32(v.Uint())) case reflect.Uint64: e.uint64(v.Uint()) case reflect.Float32: e.uint32(math.Float32bits(float32(v.Float()))) case reflect.Float64: e.uint64(math.Float64bits(v.Float())) case reflect.Complex64: x := v.Complex() e.uint32(math.Float32bits(float32(real(x)))) e.uint32(math.Float32bits(float32(imag(x)))) case reflect.Complex128: x := v.Complex() e.uint64(math.Float64bits(real(x))) e.uint64(math.Float64bits(imag(x))) } } func (d *decoder) skip(v reflect.Value) { d.offset += dataSize(v) } func (e *encoder) skip(v reflect.Value) { n := dataSize(v) clear(e.buf[e.offset : e.offset+n]) e.offset += n } // intDataSize returns the size of the data required to represent the data when encoded, // and optionally a byte slice containing the encoded data if no conversion is necessary. // It returns zero, nil if the type cannot be implemented by the fast path in Read or Write. func intDataSize(data any) (int, []byte) { switch data := data.(type) { case bool, int8, uint8, *bool, *int8, *uint8: return 1, nil case []bool: return len(data), nil case []int8: return len(data), nil case []uint8: return len(data), data case int16, uint16, *int16, *uint16: return 2, nil case []int16: return 2 * len(data), nil case []uint16: return 2 * len(data), nil case int32, uint32, *int32, *uint32: return 4, nil case []int32: return 4 * len(data), nil case []uint32: return 4 * len(data), nil case int64, uint64, *int64, *uint64: return 8, nil case []int64: return 8 * len(data), nil case []uint64: return 8 * len(data), nil case float32, *float32: return 4, nil case float64, *float64: return 8, nil case []float32: return 4 * len(data), nil case []float64: return 8 * len(data), nil } return 0, nil } // ensure grows buf to length len(buf) + n and returns the grown buffer // and a slice starting at the original length of buf (that is, buf2[len(buf):]). func ensure(buf []byte, n int) (buf2, pos []byte) { l := len(buf) buf = slices.Grow(buf, n)[:l+n] return buf, buf[l:] }