cpu_x86.go

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build 386 || amd64
     6  
     7  package cpu
     8  
     9  const CacheLinePadSize = 64
    10  
    11  // cpuid is implemented in cpu_x86.s.
    12  func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
    13  
    14  // xgetbv with ecx = 0 is implemented in cpu_x86.s.
    15  func xgetbv() (eax, edx uint32)
    16  
    17  // getGOAMD64level is implemented in cpu_x86.s. Returns number in [1,4].
    18  func getGOAMD64level() int32
    19  
    20  const (
    21  	// eax bits
    22  	cpuid_AVXVNNI = 1 << 4
    23  
    24  	// ecx bits
    25  	cpuid_SSE3            = 1 << 0
    26  	cpuid_PCLMULQDQ       = 1 << 1
    27  	cpuid_AVX512VBMI      = 1 << 1
    28  	cpuid_AVX512VBMI2     = 1 << 6
    29  	cpuid_SSSE3           = 1 << 9
    30  	cpuid_AVX512GFNI      = 1 << 8
    31  	cpuid_VAES            = 1 << 9
    32  	cpuid_AVX512VNNI      = 1 << 11
    33  	cpuid_AVX512BITALG    = 1 << 12
    34  	cpuid_FMA             = 1 << 12
    35  	cpuid_AVX512VPOPCNTDQ = 1 << 14
    36  	cpuid_SSE41           = 1 << 19
    37  	cpuid_SSE42           = 1 << 20
    38  	cpuid_POPCNT          = 1 << 23
    39  	cpuid_AES             = 1 << 25
    40  	cpuid_OSXSAVE         = 1 << 27
    41  	cpuid_AVX             = 1 << 28
    42  
    43  	// "Extended Feature Flag" bits returned in EBX for CPUID EAX=0x7 ECX=0x0
    44  	cpuid_BMI1     = 1 << 3
    45  	cpuid_AVX2     = 1 << 5
    46  	cpuid_BMI2     = 1 << 8
    47  	cpuid_ERMS     = 1 << 9
    48  	cpuid_AVX512F  = 1 << 16
    49  	cpuid_AVX512DQ = 1 << 17
    50  	cpuid_ADX      = 1 << 19
    51  	cpuid_AVX512CD = 1 << 28
    52  	cpuid_SHA      = 1 << 29
    53  	cpuid_AVX512BW = 1 << 30
    54  	cpuid_AVX512VL = 1 << 31
    55  
    56  	// "Extended Feature Flag" bits returned in ECX for CPUID EAX=0x7 ECX=0x0
    57  	cpuid_AVX512_VBMI      = 1 << 1
    58  	cpuid_AVX512_VBMI2     = 1 << 6
    59  	cpuid_GFNI             = 1 << 8
    60  	cpuid_VPCLMULQDQ       = 1 << 10 // applies to not just AVX512
    61  	cpuid_AVX512VPCLMULQDQ = 1 << 10
    62  	cpuid_AVX512_BITALG    = 1 << 12
    63  
    64  	// edx bits
    65  	cpuid_FSRM = 1 << 4
    66  	// edx bits for CPUID 0x80000001
    67  	cpuid_RDTSCP = 1 << 27
    68  )
    69  
    70  var maxExtendedFunctionInformation uint32
    71  
    72  func doinit() {
    73  	options = []option{
    74  		{Name: "adx", Feature: &X86.HasADX},
    75  		{Name: "aes", Feature: &X86.HasAES},
    76  		{Name: "erms", Feature: &X86.HasERMS},
    77  		{Name: "fsrm", Feature: &X86.HasFSRM},
    78  		{Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
    79  		{Name: "rdtscp", Feature: &X86.HasRDTSCP},
    80  		{Name: "sha", Feature: &X86.HasSHA},
    81  		{Name: "vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ},
    82  	}
    83  	level := getGOAMD64level()
    84  	if level < 2 {
    85  		// These options are required at level 2. At lower levels
    86  		// they can be turned off.
    87  		options = append(options,
    88  			option{Name: "popcnt", Feature: &X86.HasPOPCNT},
    89  			option{Name: "sse3", Feature: &X86.HasSSE3},
    90  			option{Name: "sse41", Feature: &X86.HasSSE41},
    91  			option{Name: "sse42", Feature: &X86.HasSSE42},
    92  			option{Name: "ssse3", Feature: &X86.HasSSSE3})
    93  	}
    94  	if level < 3 {
    95  		// These options are required at level 3. At lower levels
    96  		// they can be turned off.
    97  		options = append(options,
    98  			option{Name: "avx", Feature: &X86.HasAVX},
    99  			option{Name: "avx2", Feature: &X86.HasAVX2},
   100  			option{Name: "bmi1", Feature: &X86.HasBMI1},
   101  			option{Name: "bmi2", Feature: &X86.HasBMI2},
   102  			option{Name: "fma", Feature: &X86.HasFMA})
   103  	}
   104  	if level < 4 {
   105  		// These options are required at level 4. At lower levels
   106  		// they can be turned off.
   107  		options = append(options,
   108  			option{Name: "avx512f", Feature: &X86.HasAVX512F},
   109  			option{Name: "avx512cd", Feature: &X86.HasAVX512CD},
   110  			option{Name: "avx512bw", Feature: &X86.HasAVX512BW},
   111  			option{Name: "avx512dq", Feature: &X86.HasAVX512DQ},
   112  			option{Name: "avx512vl", Feature: &X86.HasAVX512VL},
   113  		)
   114  	}
   115  
   116  	maxID, _, _, _ := cpuid(0, 0)
   117  
   118  	if maxID < 1 {
   119  		osInit()
   120  		return
   121  	}
   122  
   123  	maxExtendedFunctionInformation, _, _, _ = cpuid(0x80000000, 0)
   124  
   125  	_, _, ecx1, _ := cpuid(1, 0)
   126  
   127  	X86.HasSSE3 = isSet(ecx1, cpuid_SSE3)
   128  	X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ)
   129  	X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3)
   130  	X86.HasSSE41 = isSet(ecx1, cpuid_SSE41)
   131  	X86.HasSSE42 = isSet(ecx1, cpuid_SSE42)
   132  	X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT)
   133  	X86.HasAES = isSet(ecx1, cpuid_AES)
   134  
   135  	// OSXSAVE can be false when using older Operating Systems
   136  	// or when explicitly disabled on newer Operating Systems by
   137  	// e.g. setting the xsavedisable boot option on Windows 10.
   138  	X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE)
   139  
   140  	osSupportsAVX := false
   141  	osSupportsAVX512 := false
   142  	// For XGETBV, OSXSAVE bit is required and sufficient.
   143  	if X86.HasOSXSAVE {
   144  		eax, _ := xgetbv()
   145  		// Check if XMM and YMM registers have OS support.
   146  		osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2)
   147  
   148  		// AVX512 detection does not work on Darwin,
   149  		// see https://github.com/golang/go/issues/49233
   150  		//
   151  		// Check if opmask, ZMMhi256 and Hi16_ZMM have OS support.
   152  		osSupportsAVX512 = osSupportsAVX && isSet(eax, 1<<5) && isSet(eax, 1<<6) && isSet(eax, 1<<7)
   153  	}
   154  
   155  	X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX
   156  
   157  	// The FMA instruction set extension requires both the FMA and AVX flags.
   158  	//
   159  	// Furthermore, the FMA instructions are all VEX prefixed instructions.
   160  	// VEX prefixed instructions require OSXSAVE to be enabled.
   161  	// See Intel 64 and IA-32 Architecture Software Developer’s Manual Volume 2
   162  	// Section 2.4 "AVX and SSE Instruction Exception Specification"
   163  	X86.HasFMA = isSet(ecx1, cpuid_FMA) && X86.HasAVX && X86.HasOSXSAVE
   164  
   165  	if maxID < 7 {
   166  		osInit()
   167  		return
   168  	}
   169  
   170  	eax7, ebx7, ecx7, edx7 := cpuid(7, 0)
   171  	X86.HasBMI1 = isSet(ebx7, cpuid_BMI1)
   172  	X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX
   173  	X86.HasBMI2 = isSet(ebx7, cpuid_BMI2)
   174  	X86.HasERMS = isSet(ebx7, cpuid_ERMS)
   175  	X86.HasADX = isSet(ebx7, cpuid_ADX)
   176  	X86.HasSHA = isSet(ebx7, cpuid_SHA)
   177  	X86.HasVAES = isSet(ecx7, cpuid_VAES) && X86.HasAVX
   178  	X86.HasVPCLMULQDQ = isSet(ecx7, cpuid_VPCLMULQDQ)
   179  
   180  	X86.HasAVX512F = isSet(ebx7, cpuid_AVX512F) && osSupportsAVX512
   181  	if X86.HasAVX512F {
   182  		X86.HasAVX512CD = isSet(ebx7, cpuid_AVX512CD)
   183  		X86.HasAVX512BW = isSet(ebx7, cpuid_AVX512BW)
   184  		X86.HasAVX512DQ = isSet(ebx7, cpuid_AVX512DQ)
   185  		X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL)
   186  		X86.HasAVX512GFNI = isSet(ecx7, cpuid_AVX512GFNI)
   187  		X86.HasAVX512BITALG = isSet(ecx7, cpuid_AVX512BITALG)
   188  		X86.HasAVX512VPOPCNTDQ = isSet(ecx7, cpuid_AVX512VPOPCNTDQ)
   189  		X86.HasAVX512VBMI = isSet(ecx7, cpuid_AVX512VBMI)
   190  		X86.HasAVX512VBMI2 = isSet(ecx7, cpuid_AVX512VBMI2)
   191  		X86.HasAVX512VAES = isSet(ecx7, cpuid_VAES) && X86.HasAES && isSet(ebx7, cpuid_AVX512VL)
   192  		X86.HasAVX512VNNI = isSet(ecx7, cpuid_AVX512VNNI)
   193  		X86.HasAVX512VPCLMULQDQ = isSet(ecx7, cpuid_AVX512VPCLMULQDQ)
   194  		X86.HasAVX512VBMI = isSet(ecx7, cpuid_AVX512_VBMI)
   195  		X86.HasAVX512VBMI2 = isSet(ecx7, cpuid_AVX512_VBMI2)
   196  		X86.HasGFNI = isSet(ecx7, cpuid_GFNI)
   197  		X86.HasAVX512BITALG = isSet(ecx7, cpuid_AVX512_BITALG)
   198  	}
   199  
   200  	X86.HasFSRM = isSet(edx7, cpuid_FSRM)
   201  
   202  	var maxExtendedInformation uint32
   203  	maxExtendedInformation, _, _, _ = cpuid(0x80000000, 0)
   204  
   205  	if maxExtendedInformation < 0x80000001 {
   206  		osInit()
   207  		return
   208  	}
   209  
   210  	_, _, _, edxExt1 := cpuid(0x80000001, 0)
   211  	X86.HasRDTSCP = isSet(edxExt1, cpuid_RDTSCP)
   212  
   213  	doDerived = func() {
   214  		// Rather than carefully gating on fundamental AVX-512 features, we have
   215  		// a virtual "AVX512" feature that captures F+CD+BW+DQ+VL. BW, DQ, and
   216  		// VL have a huge effect on which AVX-512 instructions are available,
   217  		// and these have all been supported on everything except the earliest
   218  		// Phi chips with AVX-512. No CPU has had CD without F, so we include
   219  		// it. GOAMD64=v4 also implies exactly this set, and these are all
   220  		// included in AVX10.1.
   221  		X86.HasAVX512 = X86.HasAVX512F && X86.HasAVX512CD && X86.HasAVX512BW && X86.HasAVX512DQ && X86.HasAVX512VL
   222  	}
   223  
   224  	if eax7 >= 1 {
   225  		eax71, _, _, _ := cpuid(7, 1)
   226  		if X86.HasAVX {
   227  			X86.HasAVXVNNI = isSet(eax71, cpuid_AVXVNNI)
   228  		}
   229  	}
   230  
   231  	osInit()
   232  }
   233  
   234  func isSet(hwc uint32, value uint32) bool {
   235  	return hwc&value != 0
   236  }
   237  
   238  // Name returns the CPU name given by the vendor.
   239  // If the CPU name can not be determined an
   240  // empty string is returned.
   241  func Name() string {
   242  	if maxExtendedFunctionInformation < 0x80000004 {
   243  		return ""
   244  	}
   245  
   246  	data := make([]byte, 0, 3*4*4)
   247  
   248  	var eax, ebx, ecx, edx uint32
   249  	eax, ebx, ecx, edx = cpuid(0x80000002, 0)
   250  	data = appendBytes(data, eax, ebx, ecx, edx)
   251  	eax, ebx, ecx, edx = cpuid(0x80000003, 0)
   252  	data = appendBytes(data, eax, ebx, ecx, edx)
   253  	eax, ebx, ecx, edx = cpuid(0x80000004, 0)
   254  	data = appendBytes(data, eax, ebx, ecx, edx)
   255  
   256  	// Trim leading spaces.
   257  	for len(data) > 0 && data[0] == ' ' {
   258  		data = data[1:]
   259  	}
   260  
   261  	// Trim tail after and including the first null byte.
   262  	for i, c := range data {
   263  		if c == '\x00' {
   264  			data = data[:i]
   265  			break
   266  		}
   267  	}
   268  
   269  	return string(data)
   270  }
   271  
   272  func appendBytes(b []byte, args ...uint32) []byte {
   273  	for _, arg := range args {
   274  		b = append(b,
   275  			byte((arg >> 0)),
   276  			byte((arg >> 8)),
   277  			byte((arg >> 16)),
   278  			byte((arg >> 24)))
   279  	}
   280  	return b
   281  }
   282
View as plain text