internal/cpu: add darwin/arm64 CPU feature detection support

Fixes #42747

Change-Id: I6b1679348c77161f075f0678818bb003fc0e8c86
Reviewed-on: https://go-review.googlesource.com/c/go/+/271989
Trust: Martin Möhrmann <moehrmann@google.com>
Run-TryBot: Martin Möhrmann <martisch@uos.de>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

src/internal/cpu/cpu_android.go

@@ -1,7 +0,0 @@
-// Copyright 2020 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cpu
-
-const GOOS = "android"

src/internal/cpu/cpu_arm64.go

@@ -6,21 +6,6 @@ package cpu
 
 const CacheLinePadSize = 64
 
-// HWCap may be initialized by archauxv and
-// should not be changed after it was initialized.
-var HWCap uint
-
-// HWCAP bits. These are exposed by Linux.
-const (
-	hwcap_AES     = 1 << 3
-	hwcap_PMULL   = 1 << 4
-	hwcap_SHA1    = 1 << 5
-	hwcap_SHA2    = 1 << 6
-	hwcap_CRC32   = 1 << 7
-	hwcap_ATOMICS = 1 << 8
-	hwcap_CPUID   = 1 << 11
-)
-
 func doinit() {
 	options = []option{
 		{Name: "aes", Feature: &ARM64.HasAES},
@@ -34,86 +19,8 @@ func doinit() {
 		{Name: "isZeus", Feature: &ARM64.IsZeus},
 	}
 
-	switch GOOS {
+	// arm64 uses different ways to detect CPU features at runtime depending on the operating system.
-	case "linux", "android":
+	osInit()
-		// HWCap was populated by the runtime from the auxiliary vector.
-		// Use HWCap information since reading aarch64 system registers
-		// is not supported in user space on older linux kernels.
-		ARM64.HasAES = isSet(HWCap, hwcap_AES)
-		ARM64.HasPMULL = isSet(HWCap, hwcap_PMULL)
-		ARM64.HasSHA1 = isSet(HWCap, hwcap_SHA1)
-		ARM64.HasSHA2 = isSet(HWCap, hwcap_SHA2)
-		ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32)
-		ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID)
-
-		// The Samsung S9+ kernel reports support for atomics, but not all cores
-		// actually support them, resulting in SIGILL. See issue #28431.
-		// TODO(elias.naur): Only disable the optimization on bad chipsets on android.
-		ARM64.HasATOMICS = isSet(HWCap, hwcap_ATOMICS) && GOOS != "android"
-
-		// Check to see if executing on a NeoverseN1 and in order to do that,
-		// check the AUXV for the CPUID bit. The getMIDR function executes an
-		// instruction which would normally be an illegal instruction, but it's
-		// trapped by the kernel, the value sanitized and then returned. Without
-		// the CPUID bit the kernel will not trap the instruction and the process
-		// will be terminated with SIGILL.
-		if ARM64.HasCPUID {
-			midr := getMIDR()
-			part_num := uint16((midr >> 4) & 0xfff)
-			implementor := byte((midr >> 24) & 0xff)
-
-			if implementor == 'A' && part_num == 0xd0c {
-				ARM64.IsNeoverseN1 = true
-			}
-			if implementor == 'A' && part_num == 0xd40 {
-				ARM64.IsZeus = true
-			}
-		}
-
-	case "freebsd":
-		// Retrieve info from system register ID_AA64ISAR0_EL1.
-		isar0 := getisar0()
-
-		// ID_AA64ISAR0_EL1
-		switch extractBits(isar0, 4, 7) {
-		case 1:
-			ARM64.HasAES = true
-		case 2:
-			ARM64.HasAES = true
-			ARM64.HasPMULL = true
-		}
-
-		switch extractBits(isar0, 8, 11) {
-		case 1:
-			ARM64.HasSHA1 = true
-		}
-
-		switch extractBits(isar0, 12, 15) {
-		case 1, 2:
-			ARM64.HasSHA2 = true
-		}
-
-		switch extractBits(isar0, 16, 19) {
-		case 1:
-			ARM64.HasCRC32 = true
-		}
-
-		switch extractBits(isar0, 20, 23) {
-		case 2:
-			ARM64.HasATOMICS = true
-		}
-	default:
-		// Other operating systems do not support reading HWCap from auxiliary vector
-		// or reading privileged aarch64 system registers in user space.
-	}
-}
-
-func extractBits(data uint64, start, end uint) uint {
-	return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
-}
-
-func isSet(hwc uint, value uint) bool {
-	return hwc&value != 0
 }
 
 func getisar0() uint64

src/internal/cpu/cpu_arm64_android.go

@@ -2,8 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !android
+// +build arm64
 
 package cpu
 
-const GOOS = "linux"
+func osInit() {
+	hwcapInit("android")
+}

src/internal/cpu/cpu_arm64_darwin.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build darwin
+// +build !ios
+
+package cpu
+
+func osInit() {
+	ARM64.HasATOMICS = sysctlEnabled([]byte("hw.optional.armv8_1_atomics\x00"))
+	ARM64.HasCRC32 = sysctlEnabled([]byte("hw.optional.armv8_crc32\x00"))
+
+	// There are no hw.optional sysctl values for the below features on Mac OS 11.0
+	// to detect their supported state dynamically. Assume the CPU features that
+	// Apple Silicon M1 supports to be available as a minimal set of features
+	// to all Go programs running on darwin/arm64.
+	ARM64.HasAES = true
+	ARM64.HasPMULL = true
+	ARM64.HasSHA1 = true
+	ARM64.HasSHA2 = true
+}
+
+//go:noescape
+func getsysctlbyname(name []byte) (int32, int32)
+
+func sysctlEnabled(name []byte) bool {
+	ret, value := getsysctlbyname(name)
+	if ret < 0 {
+		return false
+	}
+	return value > 0
+}

src/internal/cpu/cpu_arm64_freebsd.go

@@ -0,0 +1,45 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+
+package cpu
+
+func osInit() {
+	// Retrieve info from system register ID_AA64ISAR0_EL1.
+	isar0 := getisar0()
+
+	// ID_AA64ISAR0_EL1
+	switch extractBits(isar0, 4, 7) {
+	case 1:
+		ARM64.HasAES = true
+	case 2:
+		ARM64.HasAES = true
+		ARM64.HasPMULL = true
+	}
+
+	switch extractBits(isar0, 8, 11) {
+	case 1:
+		ARM64.HasSHA1 = true
+	}
+
+	switch extractBits(isar0, 12, 15) {
+	case 1, 2:
+		ARM64.HasSHA2 = true
+	}
+
+	switch extractBits(isar0, 16, 19) {
+	case 1:
+		ARM64.HasCRC32 = true
+	}
+
+	switch extractBits(isar0, 20, 23) {
+	case 2:
+		ARM64.HasATOMICS = true
+	}
+}
+
+func extractBits(data uint64, start, end uint) uint {
+	return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
+}

src/internal/cpu/cpu_arm64_hwcap.go

@@ -0,0 +1,63 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build linux
+
+package cpu
+
+// HWCap may be initialized by archauxv and
+// should not be changed after it was initialized.
+var HWCap uint
+
+// HWCAP bits. These are exposed by Linux.
+const (
+	hwcap_AES     = 1 << 3
+	hwcap_PMULL   = 1 << 4
+	hwcap_SHA1    = 1 << 5
+	hwcap_SHA2    = 1 << 6
+	hwcap_CRC32   = 1 << 7
+	hwcap_ATOMICS = 1 << 8
+	hwcap_CPUID   = 1 << 11
+)
+
+func hwcapInit(os string) {
+	// HWCap was populated by the runtime from the auxiliary vector.
+	// Use HWCap information since reading aarch64 system registers
+	// is not supported in user space on older linux kernels.
+	ARM64.HasAES = isSet(HWCap, hwcap_AES)
+	ARM64.HasPMULL = isSet(HWCap, hwcap_PMULL)
+	ARM64.HasSHA1 = isSet(HWCap, hwcap_SHA1)
+	ARM64.HasSHA2 = isSet(HWCap, hwcap_SHA2)
+	ARM64.HasCRC32 = isSet(HWCap, hwcap_CRC32)
+	ARM64.HasCPUID = isSet(HWCap, hwcap_CPUID)
+
+	// The Samsung S9+ kernel reports support for atomics, but not all cores
+	// actually support them, resulting in SIGILL. See issue #28431.
+	// TODO(elias.naur): Only disable the optimization on bad chipsets on android.
+	ARM64.HasATOMICS = isSet(HWCap, hwcap_ATOMICS) && os != "android"
+
+	// Check to see if executing on a NeoverseN1 and in order to do that,
+	// check the AUXV for the CPUID bit. The getMIDR function executes an
+	// instruction which would normally be an illegal instruction, but it's
+	// trapped by the kernel, the value sanitized and then returned. Without
+	// the CPUID bit the kernel will not trap the instruction and the process
+	// will be terminated with SIGILL.
+	if ARM64.HasCPUID {
+		midr := getMIDR()
+		part_num := uint16((midr >> 4) & 0xfff)
+		implementor := byte((midr >> 24) & 0xff)
+
+		if implementor == 'A' && part_num == 0xd0c {
+			ARM64.IsNeoverseN1 = true
+		}
+		if implementor == 'A' && part_num == 0xd40 {
+			ARM64.IsZeus = true
+		}
+	}
+}
+
+func isSet(hwc uint, value uint) bool {
+	return hwc&value != 0
+}

src/internal/cpu/cpu_arm64_linux.go

@@ -2,10 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !linux
+// +build arm64
-// +build !freebsd
+// +build linux
 // +build !android
 
 package cpu
 
-const GOOS = "other"
+func osInit() {
+	hwcapInit("linux")
+}

src/internal/cpu/cpu_arm64_other.go

@@ -0,0 +1,17 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build !linux
+// +build !freebsd
+// +build !android
+// +build !darwin ios
+
+package cpu
+
+func osInit() {
+	// Other operating systems do not support reading HWCap from auxiliary vector,
+	// reading privileged aarch64 system registers or sysctl in user space to detect
+	// CPU features at runtime.
+}

src/internal/cpu/cpu_freebsd.go

@@ -1,7 +0,0 @@
-// Copyright 2020 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cpu
-
-const GOOS = "freebsd"

src/internal/cpu/cpu_test.go

@@ -18,7 +18,7 @@ func TestMinimalFeatures(t *testing.T) {
 	// TODO: maybe do MustSupportFeatureDectection(t) ?
 	if runtime.GOARCH == "arm64" {
 		switch runtime.GOOS {
-		case "linux", "android":
+		case "linux", "android", "darwin":
 		default:
 			t.Skipf("%s/%s is not supported", runtime.GOOS, runtime.GOARCH)
 		}
@@ -38,10 +38,7 @@ func MustHaveDebugOptionsSupport(t *testing.T) {
 }
 
 func MustSupportFeatureDectection(t *testing.T) {
-	if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" {
+	// TODO: add platforms that do not have CPU feature detection support.
-		t.Skipf("CPU feature detection is not supported on %s/%s", runtime.GOOS, runtime.GOARCH)
-	}
-	// TODO: maybe there are other platforms?
 }
 
 func runDebugOptionsTest(t *testing.T, test string, options string) {

src/runtime/os_darwin.go

@@ -130,6 +130,18 @@ func osinit() {
 	physPageSize = getPageSize()
 }
 
+func sysctlbynameInt32(name []byte) (int32, int32) {
+	out := int32(0)
+	nout := unsafe.Sizeof(out)
+	ret := sysctlbyname(&name[0], (*byte)(unsafe.Pointer(&out)), &nout, nil, 0)
+	return ret, out
+}
+
+//go:linkname internal_cpu_getsysctlbyname internal/cpu.getsysctlbyname
+func internal_cpu_getsysctlbyname(name []byte) (int32, int32) {
+	return sysctlbynameInt32(name)
+}
+
 const (
 	_CTL_HW      = 6
 	_HW_NCPU     = 3

src/runtime/sys_darwin.go

@@ -360,11 +360,18 @@ func setitimer_trampoline()
 
 //go:nosplit
 //go:cgo_unsafe_args
-func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 {
+func sysctl(mib *uint32, miblen uint32, oldp *byte, oldlenp *uintptr, newp *byte, newlen uintptr) int32 {
 	return libcCall(unsafe.Pointer(funcPC(sysctl_trampoline)), unsafe.Pointer(&mib))
 }
 func sysctl_trampoline()
 
+//go:nosplit
+//go:cgo_unsafe_args
+func sysctlbyname(name *byte, oldp *byte, oldlenp *uintptr, newp *byte, newlen uintptr) int32 {
+	return libcCall(unsafe.Pointer(funcPC(sysctlbyname_trampoline)), unsafe.Pointer(&name))
+}
+func sysctlbyname_trampoline()
+
 //go:nosplit
 //go:cgo_unsafe_args
 func fcntl(fd, cmd, arg int32) int32 {
@@ -486,6 +493,7 @@ func setNonblock(fd int32) {
 //go:cgo_import_dynamic libc_kill kill "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_setitimer setitimer "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_sysctl sysctl "/usr/lib/libSystem.B.dylib"
+//go:cgo_import_dynamic libc_sysctlbyname sysctlbyname "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_fcntl fcntl "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_kqueue kqueue "/usr/lib/libSystem.B.dylib"
 //go:cgo_import_dynamic libc_kevent kevent "/usr/lib/libSystem.B.dylib"

src/runtime/sys_darwin_amd64.s

@@ -371,15 +371,27 @@ TEXT runtime·sysctl_trampoline(SB),NOSPLIT,$0
 	PUSHQ	BP
 	MOVQ	SP, BP
 	MOVL	8(DI), SI		// arg 2 miblen
-	MOVQ	16(DI), DX		// arg 3 out
+	MOVQ	16(DI), DX		// arg 3 oldp
-	MOVQ	24(DI), CX		// arg 4 size
+	MOVQ	24(DI), CX		// arg 4 oldlenp
-	MOVQ	32(DI), R8		// arg 5 dst
+	MOVQ	32(DI), R8		// arg 5 newp
-	MOVQ	40(DI), R9		// arg 6 ndst
+	MOVQ	40(DI), R9		// arg 6 newlen
 	MOVQ	0(DI), DI		// arg 1 mib
 	CALL	libc_sysctl(SB)
 	POPQ	BP
 	RET
 
+TEXT runtime·sysctlbyname_trampoline(SB),NOSPLIT,$0
+	PUSHQ	BP
+	MOVQ	SP, BP
+	MOVQ	8(DI), SI		// arg 2 oldp
+	MOVQ	16(DI), DX		// arg 3 oldlenp
+	MOVQ	24(DI), CX		// arg 4 newp
+	MOVQ	32(DI), R8		// arg 5 newlen
+	MOVQ	0(DI), DI		// arg 1 name
+	CALL	libc_sysctlbyname(SB)
+	POPQ	BP
+	RET
+
 TEXT runtime·kqueue_trampoline(SB),NOSPLIT,$0
 	PUSHQ	BP
 	MOVQ	SP, BP

src/runtime/sys_darwin_arm64.s

@@ -301,14 +301,24 @@ TEXT runtime·usleep_trampoline(SB),NOSPLIT,$0
 
 TEXT runtime·sysctl_trampoline(SB),NOSPLIT,$0
 	MOVW	8(R0), R1	// arg 2 miblen
-	MOVD	16(R0), R2	// arg 3 out
+	MOVD	16(R0), R2	// arg 3 oldp
-	MOVD	24(R0), R3	// arg 4 size
+	MOVD	24(R0), R3	// arg 4 oldlenp
-	MOVD	32(R0), R4	// arg 5 dst
+	MOVD	32(R0), R4	// arg 5 newp
-	MOVD	40(R0), R5	// arg 6 ndst
+	MOVD	40(R0), R5	// arg 6 newlen
 	MOVD	0(R0), R0	// arg 1 mib
 	BL	libc_sysctl(SB)
 	RET
 
+TEXT runtime·sysctlbyname_trampoline(SB),NOSPLIT,$0
+	MOVD	8(R0), R1	// arg 2 oldp
+	MOVD	16(R0), R2	// arg 3 oldlenp
+	MOVD	24(R0), R3	// arg 4 newp
+	MOVD	32(R0), R4	// arg 5 newlen
+	MOVD	0(R0), R0	// arg 1 name
+	BL	libc_sysctlbyname(SB)
+	RET
+
+
 TEXT runtime·kqueue_trampoline(SB),NOSPLIT,$0
 	BL	libc_kqueue(SB)
 	RET